Added logging, changed some directory structure

This commit is contained in:
2018-01-13 21:33:40 -05:00
parent f079a5f067
commit 8e72ffb917
73656 changed files with 35284 additions and 53718 deletions

View File

@@ -0,0 +1,10 @@
{
"env": {
"node": true
},
"rules": {
"strict": 0,
"curly": 0,
"quotes": 0
}
}

View File

@@ -0,0 +1,2 @@
node_modules
coverage

View File

@@ -0,0 +1,12 @@
sudo: false
language: node_js
node_js:
- "0.10"
- "0.12"
- "iojs"
script: npm run travis
before_install:
- '[ "${TRAVIS_NODE_VERSION}" != "0.10" ] || npm install -g npm'
after_success: cat ./coverage/lcov.info | node_modules/.bin/coveralls --verbose && rm -rf ./coverage

View File

@@ -0,0 +1,118 @@
# fastparse
A very simple and stupid parser, based on a statemachine and regular expressions.
It's not intended for complex languages. It's intended to easily write a simple parser for a simple language.
## Usage
Pass a description of statemachine to the constructor. The description must be in this form:
``` javascript
new Parser(description)
description is {
// The key is the name of the state
// The value is an object containing possible transitions
"state-name": {
// The key is a regular expression
// If the regular expression matches the transition is executed
// The value can be "true", a other state name or a function
"a": true,
// true will make the parser stay in the current state
"b": "other-state-name",
// a string will make the parser transit to a new state
"[cde]": function(match, index, matchLength) {
// "match" will be the matched string
// "index" will be the position in the complete string
// "matchLength" will be "match.length"
// "this" will be the "context" passed to the "parse" method"
// A new state name (string) can be returned
return "other-state-name";
},
"([0-9]+)(\\.[0-9]+)?": function(match, first, second, index, matchLength) {
// groups can be used in the regular expression
// they will match to arguments "first", "second"
},
// the parser stops when it cannot match the string anymore
// order of keys is the order in which regular expressions are matched
// if the javascript runtime preserves the order of keys in an object
// (this is not standardized, but it's a de-facto standard)
}
}
```
The statemachine is compiled down to a single regular expression per state. So basically the parsing work is delegated to the (native) regular expression logic of the javascript runtime.
``` javascript
Parser.prototype.parse(initialState: String, parsedString: String, context: Object)
```
`initialState`: state where the parser starts to parse.
`parsedString`: the string which should be parsed.
`context`: an object which can be used to save state and results. Available as `this` in transition functions.
returns `context`
## Example
``` javascript
var Parser = require("fastparse");
// A simple parser that extracts @licence ... from comments in a JS file
var parser = new Parser({
// The "source" state
"source": {
// matches comment start
"/\\*": "comment",
"//": "linecomment",
// this would be necessary for a complex language like JS
// but omitted here for simplicity
// "\"": "string1",
// "\'": "string2",
// "\/": "regexp"
},
// The "comment" state
"comment": {
"\\*/": "source",
"@licen[cs]e\\s((?:[^*\n]|\\*+[^*/\n])*)": function(match, licenseText) {
this.licences.push(licenseText.trim());
}
},
// The "linecomment" state
"linecomment": {
"\n": "source",
"@licen[cs]e\\s(.*)": function(match, licenseText) {
this.licences.push(licenseText.trim());
}
}
});
var licences = parser.parse("source", sourceCode, { licences: [] }).licences;
console.log(licences);
```
## License
MIT (http://www.opensource.org/licenses/mit-license.php)

View File

@@ -0,0 +1,108 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
function ignoreFunction() {}
function createReturningFunction(value) {
return function() {
return value;
};
}
function Parser(states) {
this.states = this.compileStates(states);
}
Parser.prototype.compileStates = function(states) {
var result = {};
Object.keys(states).forEach(function(name) {
result[name] = this.compileState(states[name], states);
}, this);
return result;
};
Parser.prototype.compileState = function(state, states) {
var regExps = [];
function iterator(str, value) {
regExps.push({
groups: Parser.getGroupCount(str),
regExp: str,
value: value
});
}
function processState(statePart) {
if(Array.isArray(statePart)) {
statePart.forEach(processState);
} else if(typeof statePart === "object") {
Object.keys(statePart).forEach(function(key) {
iterator(key, statePart[key]);
});
} else if(typeof statePart === "string") {
processState(states[statePart]);
} else {
throw new Error("Unexpected 'state' format");
}
}
processState(state);
var total = regExps.map(function(r) {
return "(" + r.regExp + ")";
}).join("|");
var actions = [];
var pos = 1;
regExps.forEach(function(r) {
var fn;
if(typeof r.value === "function") {
fn = r.value;
} else if(typeof r.value === "string") {
fn = createReturningFunction(r.value);
} else {
fn = ignoreFunction;
}
actions.push({
name: r.regExp,
fn: fn,
pos: pos,
pos2: pos + r.groups + 1
});
pos += r.groups + 1;
});
return {
regExp: new RegExp(total, "g"),
actions: actions
};
};
Parser.getGroupCount = function(regExpStr) {
return new RegExp("(" + regExpStr + ")|^$").exec("").length - 2;
};
Parser.prototype.parse = function(initialState, string, context) {
context = context || {};
var currentState = initialState;
var currentIndex = 0;
for(;;) {
var state = this.states[currentState];
var regExp = state.regExp;
regExp.lastIndex = currentIndex;
var match = regExp.exec(string);
if(!match) return context;
var actions = state.actions;
currentIndex = state.regExp.lastIndex;
for(var i = 0; i < actions.length; i++) {
var action = actions[i];
if(match[action.pos]) {
var ret = action.fn.apply(context, Array.prototype.slice.call(match, action.pos, action.pos2).concat([state.regExp.lastIndex - match[0].length, match[0].length]));
if(ret) {
if(!(ret in this.states))
throw new Error("State '" + ret + "' doesn't exist");
currentState = ret;
}
break;
}
}
}
};
module.exports = Parser;

View File

@@ -0,0 +1,66 @@
{
"_args": [
[
"fastparse@1.1.1",
"C:\\Users\\deranjer\\go\\src\\github.com\\deranjer\\goTorrent\\torrent-project"
]
],
"_from": "fastparse@1.1.1",
"_id": "fastparse@1.1.1",
"_inBundle": false,
"_integrity": "sha1-0eJkOzipTXWDtHkGDmxK/8lAcfg=",
"_location": "/css-loader/fastparse",
"_phantomChildren": {},
"_requested": {
"type": "version",
"registry": true,
"raw": "fastparse@1.1.1",
"name": "fastparse",
"escapedName": "fastparse",
"rawSpec": "1.1.1",
"saveSpec": null,
"fetchSpec": "1.1.1"
},
"_requiredBy": [
"/css-loader/css-selector-tokenizer"
],
"_resolved": "https://registry.npmjs.org/fastparse/-/fastparse-1.1.1.tgz",
"_spec": "1.1.1",
"_where": "C:\\Users\\deranjer\\go\\src\\github.com\\deranjer\\goTorrent\\torrent-project",
"author": {
"name": "Tobias Koppers @sokra"
},
"bugs": {
"url": "https://github.com/webpack/fastparse/issues"
},
"description": "A very simple and stupid parser, based on a statemachine and regular expressions.",
"devDependencies": {
"coveralls": "^2.11.2",
"eslint": "^0.21.2",
"istanbul": "^0.3.14",
"mocha": "^2.2.5",
"should": "^6.0.3"
},
"homepage": "https://github.com/webpack/fastparse",
"keywords": [
"parser",
"regexp"
],
"license": "MIT",
"main": "lib/Parser.js",
"name": "fastparse",
"repository": {
"type": "git",
"url": "git+https://github.com/webpack/fastparse.git"
},
"scripts": {
"cover": "istanbul cover node_modules/mocha/bin/_mocha",
"lint": "eslint lib",
"precover": "npm run lint",
"pretest": "npm run lint",
"publish-patch": "mocha && npm version patch && git push && git push --tags && npm publish",
"test": "mocha",
"travis": "npm run cover -- --report lcovonly"
},
"version": "1.1.1"
}

View File

@@ -0,0 +1,131 @@
/*globals describe it */
require("should");
var Parser = require("../");
var testdata = [
{
name: "simple string",
states: {
"start": {
"[d-gm-rv]+": function(match, index) {
if(!this.data) this.data = [];
this.data.push({
match: match,
index: index
});
}
}
},
string: "abcdefghijklmnopqrstuvwxyz",
expected: {
data: [
{ match: "defg", index: 3 },
{ match: "mnopqr", index: 12 },
{ match: "v", index: 21 }
]
}
},
{
name: "state switing",
states: {
"number": {
"([0-9]+)": function(match, number) {
if(!this.data) this.data = {};
this.data[this.ident] = +number;
delete this.ident;
return "start";
},
"-\\?": true,
"\\?": "start"
},
"start": {
"([a-z]+)": function(match, name) {
this.ident = name;
return "number";
}
}
},
string: "a 1 b 2 c f 3 d ? e -? 4",
expected: {
data: {
a: 1, b: 2, c: 3, e: 4
}
}
},
{
name: "state array",
states: {
"start": [
{ "a": function() { this.a = true; } },
{
"b": function() { this.b = true; },
"c": function() { this.c = true; }
}
]
},
string: "hello abc",
expected: {
a: true, b: true, c: true
}
},
{
name: "reference other states",
states: {
"start": [
{ "a": function() { this.a = true; } },
"bc"
],
"bc": {
"b": function() { this.b = true; },
"c": function() { this.c = true; }
}
},
string: "hello abc",
expected: {
a: true, b: true, c: true
}
}
];
describe("Parser", function() {
testdata.forEach(function(testcase) {
it("should parse " + testcase.name, function() {
var parser = new Parser(testcase.states);
var actual = parser.parse("start", testcase.string, {});
actual.should.be.eql(testcase.expected);
});
});
it("should default context to empty object", function() {
var parser = new Parser({
"a": {
"a": function() {
this.should.be.eql({});
}
}
});
var result = parser.parse("a", "a");
result.should.be.eql({});
});
it("should error for unexpected format", function() {
(function() {
var parser = new Parser({
"a": 123
});
return parser;
}).should.throw();
});
it("should error for not existing state", function() {
var parser = new Parser({
"a": {
"a": "b"
}
});
(function() {
return parser.parse("a", "a");
}).should.throw();
});
});