Added logging, changed some directory structure

This commit is contained in:
2018-01-13 21:33:40 -05:00
parent f079a5f067
commit 8e72ffb917
73656 changed files with 35284 additions and 53718 deletions

View File

@@ -0,0 +1,48 @@
#!/usr/bin/env node
'use strict';
var sane = require('../');
var argv = require('minimist')(process.argv.slice(2));
var execshell = require('exec-sh');
if(argv._.length === 0) {
var msg = 'Usage: sane <command> [...directory] [--glob=<filePattern>] ' +
'[--poll] [--watchman] [--dot] [--wait=<seconds>]';
console.error(msg);
process.exit();
}
var opts = {};
var command = argv._[0];
var dir = argv._[1] || process.cwd();
var waitTime = Number(argv.wait || argv.w);
var dot = argv.dot || argv.d;
var glob = argv.glob || argv.g;
var poll = argv.poll || argv.p;
var watchman = argv.watchman || argv.w;
if (dot) { opts.dot = true; }
if (glob) { opts.glob = glob; }
if (poll) { opts.poll = true; }
if (watchman) { opts.watchman = true; }
var wait = false;
var watcher = sane(dir, opts);
watcher.on('ready', function () {
console.log('Watching: ', dir + '/' + (opts.glob || ''));
execshell(command);
});
watcher.on('change', function (filepath) {
if (wait) { return; }
console.log('Change detected in:', filepath);
execshell(command);
if (waitTime > 0) {
wait = true;
setTimeout(function () {
wait = false;
}, waitTime * 1000);
}
});

View File

@@ -0,0 +1,67 @@
'use strict';
var anymatch = require('anymatch');
var minimatch = require('minimatch');
/**
* Constants
*/
exports.DEFAULT_DELAY = 100;
exports.CHANGE_EVENT = 'change';
exports.DELETE_EVENT = 'delete';
exports.ADD_EVENT = 'add';
exports.ALL_EVENT = 'all';
/**
* Assigns options to the watcher.
*
* @param {NodeWatcher|PollWatcher|WatchmanWatcher} watcher
* @param {?object} opts
* @return {boolean}
* @public
*/
exports.assignOptions = function(watcher, opts) {
opts = opts || {};
watcher.globs = opts.glob || [];
watcher.dot = opts.dot || false;
watcher.ignored = opts.ignored || false;
if (!Array.isArray(watcher.globs)) {
watcher.globs = [watcher.globs];
}
watcher.hasIgnore = Boolean(opts.ignored) &&
!(Array.isArray(opts) && opts.length > 0);
watcher.doIgnore = opts.ignored ? anymatch(opts.ignored) : function () {
return false;
};
return opts;
};
/**
* Checks a file relative path against the globs array.
*
* @param {array} globs
* @param {string} relativePath
* @return {boolean}
* @public
*/
exports.isFileIncluded = function(globs, dot, doIgnore, relativePath) {
var matched;
if (globs.length) {
for (var i = 0; i < globs.length; i++) {
if (minimatch(relativePath, globs[i], {dot: dot}) &&
!doIgnore(relativePath)) {
matched = true;
break;
}
}
} else {
// Make sure we honor the dot option if even we're not using globs.
matched = (dot || minimatch(relativePath, '**/*')) &&
!doIgnore(relativePath);
}
return matched;
};

View File

@@ -0,0 +1,376 @@
'use strict';
var fs = require('fs');
var path = require('path');
var walker = require('walker');
var common = require('./common');
var platform = require('os').platform();
var EventEmitter = require('events').EventEmitter;
var anymatch = require('anymatch');
/**
* Constants
*/
var DEFAULT_DELAY = common.DEFAULT_DELAY;
var CHANGE_EVENT = common.CHANGE_EVENT;
var DELETE_EVENT = common.DELETE_EVENT;
var ADD_EVENT = common.ADD_EVENT;
var ALL_EVENT = common.ALL_EVENT;
/**
* Export `NodeWatcher` class.
*/
module.exports = NodeWatcher;
/**
* Watches `dir`.
*
* @class NodeWatcher
* @param {String} dir
* @param {Object} opts
* @public
*/
function NodeWatcher(dir, opts) {
opts = common.assignOptions(this, opts);
this.watched = Object.create(null);
this.changeTimers = Object.create(null);
this.dirRegistery = Object.create(null);
this.root = path.resolve(dir);
this.watchdir = this.watchdir.bind(this);
this.register = this.register.bind(this);
this.watchdir(this.root);
recReaddir(
this.root,
this.watchdir,
this.register,
this.emit.bind(this, 'ready'),
this.ignored
);
}
NodeWatcher.prototype.__proto__ = EventEmitter.prototype;
/**
* Register files that matches our globs to know what to type of event to
* emit in the future.
*
* Registery looks like the following:
*
* dirRegister => Map {
* dirpath => Map {
* filename => true
* }
* }
*
* @param {string} filepath
* @return {boolean} whether or not we have registered the file.
* @private
*/
NodeWatcher.prototype.register = function(filepath) {
var relativePath = path.relative(this.root, filepath);
if (!common.isFileIncluded(
this.globs,
this.dot,
this.doIgnore,
relativePath)) {
return false;
}
var dir = path.dirname(filepath);
if (!this.dirRegistery[dir]) {
this.dirRegistery[dir] = Object.create(null);
}
var filename = path.basename(filepath);
this.dirRegistery[dir][filename] = true;
return true;
};
/**
* Removes a file from the registery.
*
* @param {string} filepath
* @private
*/
NodeWatcher.prototype.unregister = function(filepath) {
var dir = path.dirname(filepath);
if (this.dirRegistery[dir]) {
var filename = path.basename(filepath);
delete this.dirRegistery[dir][filename];
}
};
/**
* Removes a dir from the registery.
*
* @param {string} dirpath
* @private
*/
NodeWatcher.prototype.unregisterDir = function(dirpath) {
if (this.dirRegistery[dirpath]) {
delete this.dirRegistery[dirpath];
}
};
/**
* Checks if a file or directory exists in the registery.
*
* @param {string} fullpath
* @return {boolean}
* @private
*/
NodeWatcher.prototype.registered = function(fullpath) {
var dir = path.dirname(fullpath);
return this.dirRegistery[fullpath] ||
this.dirRegistery[dir] && this.dirRegistery[dir][path.basename(fullpath)];
};
/**
* Watch a directory.
*
* @param {string} dir
* @private
*/
NodeWatcher.prototype.watchdir = function(dir) {
if (this.watched[dir]) {
return;
}
var watcher = fs.watch(
dir,
{ persistent: true },
this.normalizeChange.bind(this, dir)
);
this.watched[dir] = watcher;
// Workaround Windows node issue #4337.
if (platform === 'win32') {
watcher.on('error', function(error) {
if (error.code !== 'EPERM') {
throw error;
}
});
}
if (this.root !== dir) {
this.register(dir);
}
};
/**
* Stop watching a directory.
*
* @param {string} dir
* @private
*/
NodeWatcher.prototype.stopWatching = function(dir) {
if (this.watched[dir]) {
this.watched[dir].close();
delete this.watched[dir];
}
};
/**
* End watching.
*
* @public
*/
NodeWatcher.prototype.close = function(callback) {
Object.keys(this.watched).forEach(this.stopWatching, this);
this.removeAllListeners();
if (typeof callback === 'function') {
setImmediate(callback.bind(null, null, true));
}
};
/**
* On some platforms, as pointed out on the fs docs (most likely just win32)
* the file argument might be missing from the fs event. Try to detect what
* change by detecting if something was deleted or the most recent file change.
*
* @param {string} dir
* @param {string} event
* @param {string} file
* @public
*/
NodeWatcher.prototype.detectChangedFile = function(dir, event, callback) {
if (!this.dirRegistery[dir]) {
return;
}
var found = false;
var closest = {mtime: 0};
var c = 0;
Object.keys(this.dirRegistery[dir]).forEach(function(file, i, arr) {
fs.lstat(path.join(dir, file), function(error, stat) {
if (found) {
return;
}
if (error) {
if (error.code === 'ENOENT' ||
(platform === 'win32' && error.code === 'EPERM')) {
found = true;
callback(file);
} else {
this.emit('error', error);
}
} else {
if (stat.mtime > closest.mtime) {
stat.file = file;
closest = stat;
}
if (arr.length === ++c) {
callback(closest.file);
}
}
}.bind(this));
}, this);
};
/**
* Normalize fs events and pass it on to be processed.
*
* @param {string} dir
* @param {string} event
* @param {string} file
* @public
*/
NodeWatcher.prototype.normalizeChange = function(dir, event, file) {
if (!file) {
this.detectChangedFile(dir, event, function(actualFile) {
if (actualFile) {
this.processChange(dir, event, actualFile);
}
}.bind(this));
} else {
this.processChange(dir, event, path.normalize(file));
}
};
/**
* Process changes.
*
* @param {string} dir
* @param {string} event
* @param {string} file
* @public
*/
NodeWatcher.prototype.processChange = function(dir, event, file) {
var fullPath = path.join(dir, file);
var relativePath = path.join(path.relative(this.root, dir), file);
fs.lstat(fullPath, function(error, stat) {
if (error && error.code !== 'ENOENT') {
this.emit('error', error);
} else if (!error && stat.isDirectory()) {
// win32 emits usless change events on dirs.
if (event !== 'change') {
this.watchdir(fullPath);
if (common.isFileIncluded(
this.globs,
this.dot,
this.doIgnore,
relativePath)) {
this.emitEvent(ADD_EVENT, relativePath, stat);
}
}
} else {
var registered = this.registered(fullPath);
if (error && error.code === 'ENOENT') {
this.unregister(fullPath);
this.stopWatching(fullPath);
this.unregisterDir(fullPath);
if (registered) {
this.emitEvent(DELETE_EVENT, relativePath);
}
} else if (registered) {
this.emitEvent(CHANGE_EVENT, relativePath, stat);
} else {
if (this.register(fullPath)) {
this.emitEvent(ADD_EVENT, relativePath, stat);
}
}
}
}.bind(this));
};
/**
* Triggers a 'change' event after debounding it to take care of duplicate
* events on os x.
*
* @private
*/
NodeWatcher.prototype.emitEvent = function(type, file, stat) {
var key = type + '-' + file;
var addKey = ADD_EVENT + '-' + file;
if (type === CHANGE_EVENT && this.changeTimers[addKey]) {
// Ignore the change event that is immediately fired after an add event.
// (This happens on Linux).
return;
}
clearTimeout(this.changeTimers[key]);
this.changeTimers[key] = setTimeout(function() {
delete this.changeTimers[key];
this.emit(type, file, this.root, stat);
this.emit(ALL_EVENT, type, file, this.root, stat);
}.bind(this), DEFAULT_DELAY);
};
/**
* Traverse a directory recursively calling `callback` on every directory.
*
* @param {string} dir
* @param {function} dirCallback
* @param {function} fileCallback
* @param {function} endCallback
* @param {*} ignored
* @private
*/
function recReaddir(dir, dirCallback, fileCallback, endCallback, ignored) {
walker(dir)
.filterDir(function(currentDir) {
return !anymatch(ignored, currentDir);
})
.on('dir', normalizeProxy(dirCallback))
.on('file', normalizeProxy(fileCallback))
.on('end', function() {
if (platform === 'win32') {
setTimeout(endCallback, 1000);
} else {
endCallback();
}
});
}
/**
* Returns a callback that when called will normalize a path and call the
* original callback
*
* @param {function} callback
* @return {function}
* @private
*/
function normalizeProxy(callback) {
return function(filepath) {
return callback(path.normalize(filepath));
};
}

View File

@@ -0,0 +1,118 @@
'use strict';
var fs = require('fs');
var path = require('path');
var watch = require('watch');
var common = require('./common');
var EventEmitter = require('events').EventEmitter;
/**
* Constants
*/
var DEFAULT_DELAY = common.DEFAULT_DELAY;
var CHANGE_EVENT = common.CHANGE_EVENT;
var DELETE_EVENT = common.DELETE_EVENT;
var ADD_EVENT = common.ADD_EVENT;
var ALL_EVENT = common.ALL_EVENT;
/**
* Export `PollWatcher` class.
*/
module.exports = PollWatcher;
/**
* Watches `dir`.
*
* @class PollWatcher
* @param String dir
* @param {Object} opts
* @public
*/
function PollWatcher(dir, opts) {
opts = common.assignOptions(this, opts);
this.watched = Object.create(null);
this.root = path.resolve(dir);
watch.createMonitor(
this.root,
{ interval: opts.interval || DEFAULT_DELAY,
filter: this.filter.bind(this)
},
this.init.bind(this)
);
}
PollWatcher.prototype.__proto__ = EventEmitter.prototype;
/**
* Given a fullpath of a file or directory check if we need to watch it.
*
* @param {string} filepath
* @param {object} stat
* @private
*/
PollWatcher.prototype.filter = function(filepath, stat) {
return stat.isDirectory() || common.isFileIncluded(
this.globs,
this.dot,
this.doIgnore,
path.relative(this.root, filepath)
);
};
/**
* Initiate the polling file watcher with the event emitter passed from
* `watch.watchTree`.
*
* @param {EventEmitter} monitor
* @public
*/
PollWatcher.prototype.init = function(monitor) {
this.watched = monitor.files;
monitor.on('changed', this.emitEvent.bind(this, CHANGE_EVENT));
monitor.on('removed', this.emitEvent.bind(this, DELETE_EVENT));
monitor.on('created', this.emitEvent.bind(this, ADD_EVENT));
// 1 second wait because mtime is second-based.
setTimeout(this.emit.bind(this, 'ready'), 1000);
};
/**
* Transform and emit an event comming from the poller.
*
* @param {EventEmitter} monitor
* @public
*/
PollWatcher.prototype.emitEvent = function(type, file, stat) {
file = path.relative(this.root, file);
if (type === DELETE_EVENT) {
// Matching the non-polling API
stat = null;
}
this.emit(type, file, this.root, stat);
this.emit(ALL_EVENT, type, file, this.root, stat);
};
/**
* End watching.
*
* @public
*/
PollWatcher.prototype.close = function(callback) {
Object.keys(this.watched).forEach(function(filepath) {
fs.unwatchFile(filepath);
});
this.removeAllListeners();
if (typeof callback === 'function') {
setImmediate(callback.bind(null, null, true));
}
};

View File

@@ -0,0 +1,47 @@
'use strict';
var RECRAWL_WARNINGS = []; // shared structure, one per process.
var REG = /Recrawled this watch (\d+) times, most recently because:\n([^:]+)/;
module.exports = RecrawlWarning;
function RecrawlWarning(root, count) {
this.root = root;
this.count = count;
}
RecrawlWarning.RECRAWL_WARNINGS = RECRAWL_WARNINGS;
RecrawlWarning.REGEXP = REG;
RecrawlWarning.findByRoot = function(root) {
for (var i = 0; i < RECRAWL_WARNINGS.length; i++) {
var warning = RECRAWL_WARNINGS[i];
if (warning.root === root) {
return warning;
}
}
};
RecrawlWarning.isRecrawlWarningDupe = function(warningMessage) {
if (typeof warningMessage !== 'string') { return false; }
var match = warningMessage.match(REG);
if (!match) { return false; }
var count = Number(match[1]);
var root = match[2];
var warning = RecrawlWarning.findByRoot(root);
if (warning) {
// only keep the highest count, assume count to either stay the same or
// increase.
if (warning.count >= count ) {
return true;
} else {
// update the existing warning to the latest (highest) count
warning.count = count;
return false;
}
} else {
RECRAWL_WARNINGS.push(new RecrawlWarning(root, count));
return false;
}
};

View File

@@ -0,0 +1,317 @@
'use strict';
var fs = require('fs');
var path = require('path');
var assert = require('assert');
var common = require('./common');
var watchman = require('fb-watchman');
var EventEmitter = require('events').EventEmitter;
var RecrawlWarning = require('./utils/recrawl-warning-dedupe');
/**
* Constants
*/
var CHANGE_EVENT = common.CHANGE_EVENT;
var DELETE_EVENT = common.DELETE_EVENT;
var ADD_EVENT = common.ADD_EVENT;
var ALL_EVENT = common.ALL_EVENT;
var SUB_NAME = 'sane-sub';
/**
* Export `WatchmanWatcher` class.
*/
module.exports = WatchmanWatcher;
/**
* Watches `dir`.
*
* @class PollWatcher
* @param String dir
* @param {Object} opts
* @public
*/
function WatchmanWatcher(dir, opts) {
opts = common.assignOptions(this, opts);
this.root = path.resolve(dir);
this.init();
}
WatchmanWatcher.prototype.__proto__ = EventEmitter.prototype;
/**
* Run the watchman `watch` command on the root and subscribe to changes.
*
* @private
*/
WatchmanWatcher.prototype.init = function() {
if (this.client) {
this.client.removeAllListeners();
}
var self = this;
this.client = new watchman.Client();
this.client.on('error', function(error) {
self.emit('error', error);
});
this.client.on('subscription', this.handleChangeEvent.bind(this));
this.client.on('end', function() {
console.warn('[sane] Warning: Lost connection to watchman, reconnecting..');
self.init();
});
this.watchProjectInfo = null;
function getWatchRoot() {
return self.watchProjectInfo ? self.watchProjectInfo.root : self.root;
}
function onCapability(error, resp) {
if (handleError(self, error)) {
// The Watchman watcher is unusable on this system, we cannot continue
return;
}
handleWarning(resp);
self.capabilities = resp.capabilities;
if (self.capabilities.relative_root) {
self.client.command(
['watch-project', getWatchRoot()], onWatchProject
);
} else {
self.client.command(['watch', getWatchRoot()], onWatch);
}
}
function onWatchProject(error, resp) {
if (handleError(self, error)) {
return;
}
handleWarning(resp);
self.watchProjectInfo = {
root: resp.watch,
relativePath: resp.relative_path ? resp.relative_path : ''
};
self.client.command(['clock', getWatchRoot()], onClock);
}
function onWatch(error, resp) {
if (handleError(self, error)) {
return;
}
handleWarning(resp);
self.client.command(['clock', getWatchRoot()], onClock);
}
function onClock(error, resp) {
if (handleError(self, error)) {
return;
}
handleWarning(resp);
var options = {
fields: ['name', 'exists', 'new'],
since: resp.clock
};
// If the server has the wildmatch capability available it supports
// the recursive **/*.foo style match and we can offload our globs
// to the watchman server. This saves both on data size to be
// communicated back to us and compute for evaluating the globs
// in our node process.
if (self.capabilities.wildmatch) {
if (self.globs.length === 0) {
if (!self.dot) {
// Make sure we honor the dot option if even we're not using globs.
options.expression = ['match', '**', 'wholename', {
includedotfiles: false
}];
}
} else {
options.expression = ['anyof'];
for (var i in self.globs) {
options.expression.push(['match', self.globs[i], 'wholename', {
includedotfiles: self.dot
}]);
}
}
}
if (self.capabilities.relative_root) {
options.relative_root = self.watchProjectInfo.relativePath;
}
self.client.command(
['subscribe', getWatchRoot(), SUB_NAME, options],
onSubscribe
);
}
function onSubscribe(error, resp) {
if (handleError(self, error)) {
return;
}
handleWarning(resp);
self.emit('ready');
}
self.client.capabilityCheck({
optional:['wildmatch', 'relative_root']
},
onCapability);
};
/**
* Handles a change event coming from the subscription.
*
* @param {Object} resp
* @private
*/
WatchmanWatcher.prototype.handleChangeEvent = function(resp) {
assert.equal(resp.subscription, SUB_NAME, 'Invalid subscription event.');
if (Array.isArray(resp.files)) {
resp.files.forEach(this.handleFileChange, this);
}
};
/**
* Handles a single change event record.
*
* @param {Object} changeDescriptor
* @private
*/
WatchmanWatcher.prototype.handleFileChange = function(changeDescriptor) {
var self = this;
var absPath;
var relativePath;
if (this.capabilities.relative_root) {
relativePath = changeDescriptor.name;
absPath = path.join(
this.watchProjectInfo.root,
this.watchProjectInfo.relativePath,
relativePath
);
} else {
absPath = path.join(this.root, changeDescriptor.name);
relativePath = changeDescriptor.name;
}
if (!(self.capabilities.wildmatch && !this.hasIgnore) &&
!common.isFileIncluded(
this.globs,
this.dot,
this.doIgnore,
relativePath)) {
return;
}
if (!changeDescriptor.exists) {
self.emitEvent(DELETE_EVENT, relativePath, self.root);
} else {
fs.lstat(absPath, function(error, stat) {
// Files can be deleted between the event and the lstat call
// the most reliable thing to do here is to ignore the event.
if (error && error.code === 'ENOENT') {
return;
}
if (handleError(self, error)) {
return;
}
var eventType = changeDescriptor.new ? ADD_EVENT : CHANGE_EVENT;
// Change event on dirs are mostly useless.
if (!(eventType === CHANGE_EVENT && stat.isDirectory())) {
self.emitEvent(eventType, relativePath, self.root, stat);
}
});
}
};
/**
* Dispatches the event.
*
* @param {string} eventType
* @param {string} filepath
* @param {string} root
* @param {fs.Stat} stat
* @private
*/
WatchmanWatcher.prototype.emitEvent = function(
eventType,
filepath,
root,
stat
) {
this.emit(eventType, filepath, root, stat);
this.emit(ALL_EVENT, eventType, filepath, root, stat);
};
/**
* Closes the watcher.
*
* @param {function} callback
* @private
*/
WatchmanWatcher.prototype.close = function(callback) {
this.client.removeAllListeners();
this.client.end();
callback && callback(null, true);
};
/**
* Handles an error and returns true if exists.
*
* @param {WatchmanWatcher} self
* @param {Error} error
* @private
*/
function handleError(self, error) {
if (error != null) {
self.emit('error', error);
return true;
} else {
return false;
}
}
/**
* Handles a warning in the watchman resp object.
*
* @param {object} resp
* @private
*/
function handleWarning(resp) {
if ('warning' in resp) {
if (RecrawlWarning.isRecrawlWarningDupe(resp.warning)) {
return true;
}
console.warn(resp.warning);
return true;
} else {
return false;
}
}