v/human2regex
1
0
Fork 0
mirror of https://github.com/pdemian/human2regex.git synced 2025-05-15 20:10:19 -07:00
Code Issues Projects Releases Wiki Activity

Added tests and enforced a stricter eslint

Browse Source
This commit is contained in:
Patrick Demian 2020-10-29 10:37:56 -04:00
parent df11fc82a3
commit 91d1b37322
11 changed files with 4400 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
.eslintrc.json
View File

@ -10,7 +10,8 @@
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
"sourceType": "module",
"project": "./../tsconfig.json"
},
"plugins": [
"@typescript-eslint"
@ -18,7 +19,40 @@
"rules": {
"@typescript-eslint/no-this-alias": "off",
"@typescript-eslint/no-inferrable-types": "off",
"@typescript-eslint/explicit-function-return-type": "off",
"@typescript-eslint/explicit-function-return-type": [
"warn",
{ "allowExpressions": true }
],
"@typescript-eslint/no-shadow": "error",
//"@typescript-eslint/prefer-readonly-parameter-types": "warn",
//"@typescript-eslint/prefer-readonly": "warn",
"@typescript-eslint/prefer-optional-chain": "error",
"@typescript-eslint/prefer-for-of": "error",
"@typescript-eslint/no-unnecessary-condition": "error",
"@typescript-eslint/no-throw-literal": "error",
"@typescript-eslint/consistent-type-assertions": "error",
"@typescript-eslint/brace-style": [
"error",
"stroustrup"
],
"@typescript-eslint/comma-spacing": [
"error",
{ "before": false, "after": true }
],
"@typescript-eslint/keyword-spacing": "error",
"@typescript-eslint/naming-convention": [
"error",
{ "selector": "default", "format": [ "snake_case", "PascalCase" ] },
{ "selector": "property", "format": [ "camelCase", "snake_case", "PascalCase" ] },
{ "selector": [ "function", "method"], "format": [ "camelCase", "UPPER_CASE" ] },
{ "selector": "typeLike", "format": [ "PascalCase" ] }
],
"@typescript-eslint/quotes": [
"warn",
"double",
{ "avoidEscape": true }
],
"camelcase": "off",
"no-magic-numbers": [
"warn",
{ "ignoreArrayIndexes": true, "ignore": [-1,0,1,2,3,4,5,6,7,8,9]}
@ -47,14 +81,8 @@
],
"no-shadow": "off",
"no-undefined": "error",
"brace-style": [
"error",
"stroustrup"
],
"comma-spacing": [
"error",
{ "before": false, "after": true }
],
"brace-style": "off",
"comma-spacing": "off",
"array-bracket-spacing": [
"error",
"always"
@ -63,6 +91,7 @@
"error",
"last"
],
"keyword-spacing": "off",
"func-style": [
"error",
"declaration"
@ -84,19 +113,11 @@
"error",
"always"
],
/*"indent": [
"error",
4
],*/
"linebreak-style": [
"error",
"windows"
],
"quotes": [
"warn",
"double",
{ "avoidEscape": true }
],
"quotes": "off",
"semi": [
"error",
"always"

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
# Node build artifacts
node_modules/
coverage/
npm-debug.log

4
docs/bundle.min.js vendored
View File

File diff suppressed because one or more lines are too long

16
jest.config.ts Normal file
View File

@ -0,0 +1,16 @@
/*
* For a detailed explanation regarding each configuration property and type check, visit:
* https://jestjs.io/docs/en/configuration.html
*/
export default {
transform: { "^.+\\.ts$": "ts-jest" },
testEnvironment: "node",
collectCoverage: true,
coverageDirectory: "coverage",
coveragePathIgnorePatterns: [ "/node_modules/", "/docs/" ],
coverageProvider: "v8",
testRegex: "/tests/.*\\.spec\\.(ts)$",
moduleFileExtensions: [ "ts", "js" ],
verbose: true
};

4268
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

10
package.json
View File

@ -6,7 +6,8 @@
"devDependencies": {
"@types/glob": "^7.1.3",
"@types/html-minifier": "^3.5.3",
"@types/jquery": "^3.5.3",
"@types/jest": "^26.0.15",
"@types/jquery": "^3.5.4",
"@types/mustache": "^4.0.1",
"@typescript-eslint/eslint-plugin": "^4.4.0",
"@typescript-eslint/parser": "^4.4.0",
@ -16,17 +17,20 @@
"eslint": "^7.11.0",
"glob": "^7.1.6",
"html-minifier": "^4.0.0",
"jest": "^26.6.1",
"mini-css-extract-plugin": "^1.0.0",
"mustache": "^4.0.1",
"optimize-css-assets-webpack-plugin": "^5.0.4",
"ts-jest": "^26.4.3",
"ts-loader": "^8.0.4",
"typescript": "^4.0.3",
"ts-node": "^9.0.0",
"typescript": "^4.0.5",
"webpack": "^4.44.2",
"webpack-cli": "^3.3.12"
},
"scripts": {
"build": "webpack --config webpack.config.js",
"test": "echo \"Error: no test specified\" && exit 1"
"test": "eslint && jest"
},
"keywords": [
"regex"

96
src/lexer.ts
View File

@ -29,10 +29,10 @@ export class Human2RegexLexer {
Human2RegexLexer.already_init = true;
this.set_options(options);
this.setOptions(options);
}
public set_options(options: Human2RegexLexerOptions) : void {
public setOptions(options: Human2RegexLexerOptions) : void {
this.options = options;
let indent_regex: RegExp | null = null;
@ -55,7 +55,7 @@ export class Human2RegexLexer {
this.lexer = new Lexer(AllTokens, { ensureOptimizations: true, skipValidations: options.skip_validations });
}
private lex_error(token: IToken) : ILexingError {
private lexError(token: IToken) : ILexingError {
return {
offset: token.startOffset,
line: token.startLine ?? NaN,
@ -66,75 +66,75 @@ export class Human2RegexLexer {
}
public tokenize(text: string) : ILexingResult {
const lexResult = this.lexer.tokenize(text);
const lex_result = this.lexer.tokenize(text);
if (lexResult.tokens.length === 0) {
return lexResult;
if (lex_result.tokens.length === 0) {
return lex_result;
}
// create Outdents
const tokens: IToken[] = [];
const indentStack = [ 0 ];
const indent_stack = [ 0 ];
let currIndentLevel = 0;
let startOfLine = true;
let hadIndents = false;
let curr_indent_level = 0;
let start_of_line = true;
let had_indents = false;
for (let i = 0; i < lexResult.tokens.length; i++) {
for (let i = 0; i < lex_result.tokens.length; i++) {
// EoL? check for indents next (by setting startOfLine = true)
if (lexResult.tokens[i].tokenType === EndOfLine) {
if(tokens.length === 0 || tokens[tokens.length-1].tokenType === EndOfLine) {
if (lex_result.tokens[i].tokenType === EndOfLine) {
if (tokens.length === 0 || tokens[tokens.length-1].tokenType === EndOfLine) {
// Ignore multiple EOLs and ignore first EOL
}
else {
startOfLine = true;
tokens.push(lexResult.tokens[i]);
start_of_line = true;
tokens.push(lex_result.tokens[i]);
}
}
// start with 1 indent. Append all other indents
else if (lexResult.tokens[i].tokenType === Indent) {
hadIndents = true;
currIndentLevel = 1;
else if (lex_result.tokens[i].tokenType === Indent) {
had_indents = true;
curr_indent_level = 1;
const start_token = lexResult.tokens[i];
let length = lexResult.tokens[i].image.length;
const start_token = lex_result.tokens[i];
let length = lex_result.tokens[i].image.length;
// grab all the indents (and their length)
while (lexResult.tokens.length > i && lexResult.tokens[i+1].tokenType === Indent) {
currIndentLevel++;
while (lex_result.tokens.length > i && lex_result.tokens[i+1].tokenType === Indent) {
curr_indent_level++;
i++;
length += lexResult.tokens[i].image.length;
length += lex_result.tokens[i].image.length;
}
start_token.endOffset = start_token.startOffset + length;
start_token.endColumn = lexResult.tokens[i].endColumn;
start_token.endColumn = lex_result.tokens[i].endColumn;
// must be the same line
//start_token.endLine = lexResult.tokens[i].endLine;
//start_token.endLine = lex_result.tokens[i].endLine;
// are we an empty line?
if (lexResult.tokens.length > i && lexResult.tokens[i+1].tokenType === EndOfLine) {
if (lex_result.tokens.length > i && lex_result.tokens[i+1].tokenType === EndOfLine) {
// Ignore all indents AND newline
// continue;
}
else if (!startOfLine || (currIndentLevel > last(indentStack) + 1)) {
lexResult.errors.push(this.lex_error(start_token));
else if (!start_of_line || (curr_indent_level > last(indent_stack) + 1)) {
lex_result.errors.push(this.lexError(start_token));
}
else if (currIndentLevel > last(indentStack)) {
indentStack.push(currIndentLevel);
else if (curr_indent_level > last(indent_stack)) {
indent_stack.push(curr_indent_level);
tokens.push(start_token);
}
else if (currIndentLevel < last(indentStack)) {
const index = findLastIndex(indentStack, currIndentLevel);
else if (curr_indent_level < last(indent_stack)) {
const index = findLastIndex(indent_stack, curr_indent_level);
if (index < 0) {
lexResult.errors.push(this.lex_error(start_token));
lex_result.errors.push(this.lexError(start_token));
}
else {
const numberOfDedents = indentStack.length - index - 1;
const number_of_dedents = indent_stack.length - index - 1;
for(let i = 0; i < numberOfDedents; i++) {
indentStack.pop();
for (let j = 0; j < number_of_dedents; j++) {
indent_stack.pop();
tokens.push(createTokenInstance(Outdent, "", start_token.startOffset, start_token.startOffset + length, start_token.startLine ?? NaN, start_token.endLine ?? NaN, start_token.startColumn ?? NaN, (start_token.startColumn ?? NaN) + length));
}
}
@ -145,35 +145,35 @@ export class Human2RegexLexer {
}
}
else {
if(startOfLine && !hadIndents) {
const tok = lexResult.tokens[i];
if (start_of_line && !had_indents) {
const tok = lex_result.tokens[i];
//add remaining Outdents
while (indentStack.length > 1) {
indentStack.pop();
while (indent_stack.length > 1) {
indent_stack.pop();
tokens.push(createTokenInstance(Outdent, "", tok.startOffset, tok.startOffset, tok.startLine ?? NaN, NaN, tok.startColumn ?? NaN, NaN));
}
}
startOfLine = false;
hadIndents = false;
tokens.push(lexResult.tokens[i]);
start_of_line = false;
had_indents = false;
tokens.push(lex_result.tokens[i]);
}
}
const tok = last(tokens);
// Do we have an EOL marker at the end?
if(tok.tokenType !== EndOfLine) {
if (tok.tokenType !== EndOfLine) {
tokens.push(createTokenInstance(EndOfLine, "\n", tok.endOffset ?? NaN, tok.endOffset ?? NaN, tok.startLine ?? NaN, NaN, tok.startColumn ?? NaN, NaN));
}
//add remaining Outdents
while (indentStack.length > 1) {
indentStack.pop();
while (indent_stack.length > 1) {
indent_stack.pop();
tokens.push(createTokenInstance(Outdent, "", tok.endOffset ?? NaN, tok.endOffset ?? NaN, tok.startLine ?? NaN, NaN, tok.startColumn ?? NaN, NaN));
}
lexResult.tokens = tokens;
return lexResult;
lex_result.tokens = tokens;
return lex_result;
}
}

14
src/parser.ts
View File

@ -3,8 +3,20 @@
import { CstParser, CstNode, IOrAlt } from "chevrotain";
import * as T from "./tokens";
export enum RobotLanguage {
JS,
Perl,
DotNet,
Java
}
export enum HumanLanguage {
English,
/* Todo: Humans speak more than just english! */
}
export class Human2RegexParserOptions {
constructor(public skip_validations: boolean = false) {
constructor(public skip_validations: boolean = false, public robot_language: RobotLanguage = RobotLanguage.JS, public human_language: HumanLanguage = HumanLanguage.English) {
/* empty */
}
}

6
tests/lexer.spec.ts Normal file
View File

@ -0,0 +1,6 @@
describe("calculate", function() {
it("add", function() {
const result = 5 + 2;
expect(result).toBe(7);
});
});

2
tsconfig.json
View File

@ -24,7 +24,7 @@
"strict": true, /* Enable all strict type-checking options. */
"noImplicitAny": false, /* Raise error on expressions and declarations with an implied 'any' type. */
"strictNullChecks": true, /* Enable strict null checks. */
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
"strictFunctionTypes": true, /* Enable strict checking of function types. */
// "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
// "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */

6
webpack.config.js
View File

@ -3,7 +3,7 @@
const path = require("path");
const { glob } = require("glob");
const { render } = require("mustache");
const { readFileSync, writeFileSync } = require("fs");
const { readFileSync, writeFileSync, existsSync, mkdirSync } = require("fs");
const { minify } = require("html-minifier");
const CopyPlugin = require("copy-webpack-plugin");
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
@ -30,6 +30,10 @@ const config = {
function build_mustache() {
if (!existsSync(config.dst)){
mkdirSync(config.dst);
}
read_json_file = (filename) => JSON.parse(readFileSync(filename), "utf8");
compress_html = (input) => config.prod ? minify(input, config.compression_config.html) : input;