From 0601dc20ca8f2b3fe15d45a17812b31c2fca041f Mon Sep 17 00:00:00 2001 From: Patrick Demian Date: Mon, 26 Oct 2020 03:26:35 -0400 Subject: [PATCH] Last push before I break something (maybe) --- package.json | 3 +- src/parser.ts | 55 ++++++++++++-- src/tokenizer.ts | 37 ++++----- src/tokens.ts | 84 --------------------- webpack.config.js => webpack.full.config.js | 6 +- webpack.partial.config.js | 32 ++++++++ 6 files changed, 106 insertions(+), 111 deletions(-) delete mode 100644 src/tokens.ts rename webpack.config.js => webpack.full.config.js (93%) create mode 100644 webpack.partial.config.js diff --git a/package.json b/package.json index b24a8d7..b6489fd 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "webpack-cli": "^3.3.12" }, "scripts": { - "build": "webpack --config webpack.config.js", + "build": "webpack --config webpack.full.config.js", + "partial": "webpack --config webpack.partial.config.js", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [ diff --git a/src/parser.ts b/src/parser.ts index 8aaa36f..018a005 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -9,7 +9,7 @@ export class Human2RegexParser extends CstParser { const $ = this; - const Count = $.RULE("Count", () => { + const Number = $.RULE("Number", () => { $.OR([ { ALT: () => $.CONSUME(T.One) }, { ALT: () => $.CONSUME(T.Two) }, @@ -22,13 +22,49 @@ export class Human2RegexParser extends CstParser { { ALT: () => $.CONSUME(T.Nine) }, { ALT: () => $.CONSUME(T.Ten) }, { ALT: () => $.CONSUME(T.Zero) }, + { ALT: () => $.CONSUME(T.NumberLiteral) }, ]); }); - const MatchStatement = $.RULE("MatchStatement", () => { - $.OPTION(() => { - $.CONSUME(T.Optional); - }); + // 1, 1..2, between 1 and/to 2 inclusively/exclusively + const Count = $.RULE("Count", () => { + $.OR([ + { ALT: () => { + $.OPTION(() => $.CONSUME(T.Exactly)); + $.SUBRULE(Number); + }}, + { ALT: () => { + $.OPTION(() => $.CONSUME(T.From)); + $.SUBRULE(Number); + $.OR([ + { ALT: () => $.CONSUME(T.OrMore) }, + { ALT: () => { + $.CONSUME(T.To); + $.SUBRULE(Number); + }} + ]); + }}, + + { ALT: () => { + $.CONSUME(T.Between); + $.SUBRULE(Number); + $.OR([ + { ALT: () => $.CONSUME(T.To) }, + { ALT: () => $.CONSUME(T.And) } + ]); + $.SUBRULE(Number); + $.OPTION(() => { + $.OR([ + { ALT: () => $.CONSUME(T.Inclusive) }, + { ALT: () => $.CONSUME(T.Exclusive) } + ]); + }); + }} + ]); + }); + + const MatchStatement = $.RULE("Match Statement", () => { + $.OPTION(() => $.CONSUME(T.Optional)); $.CONSUME(T.Match); $.OPTION(() => { $.SUBRULE(Count); @@ -41,7 +77,7 @@ export class Human2RegexParser extends CstParser { }); }); - const UsingStatement = $.RULE("UsingStatement", () => { + const UsingStatement = $.RULE("Using Statement", () => { $.CONSUME(T.Using); $.AT_LEAST_ONE_SEP({ SEP: T.And, @@ -57,6 +93,13 @@ export class Human2RegexParser extends CstParser { ]); } }); + }); + + const Statement = $.RULE("Statement", () => { + $.OR([ + { ALT: () => $.SUBRULE(MatchStatement) }, + { ALT: () => $.SUBRULE(UsingStatement) } + ]); $.OPTION(() => $.CONSUME(T.EndOfLine)); }); diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 68d1842..6536442 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -20,13 +20,12 @@ export const Then = createToken({name: "Then", pattern: /then/i }); export const Anything = createToken({name: "Anything", pattern: /(any|anything|any thing)(s)?/i}); export const Of = createToken({name: "Of", pattern: /of/i}); export const Or = createToken({name: "Or", pattern: /or/i}); -export const And = createToken({name: "And", pattern: /and/i}); -export const Word = createToken({name: "WordSpecifier", pattern: /word(s)?/i}); -export const Digit = createToken({name: "DigitSpecifier", pattern: /digit(s)?/i}); -export const Character = createToken({name: "CharacterSpecifier", pattern: /character(s)?/i}); -export const Whitespace = createToken({name: "WhitespaceSpecifier", pattern: /(white space|whitespace)(s)?/i}); +export const And = createToken({name: "And", pattern: /and|,/i}); +export const Word = createToken({name: "Word Specifier", pattern: /word(s)?/i}); +export const Digit = createToken({name: "Digit Specifier", pattern: /digit(s)?/i}); +export const Character = createToken({name: "Character Specifier", pattern: /character(s)?/i}); +export const Whitespace = createToken({name: "Whitespace Specifier", pattern: /(white space|whitespace)(s)?/i}); export const Number = createToken({name: "NumberSpecifier", pattern: /number(s)?/i}); -export const Multiple = createToken({name: "Multiple", pattern: /multiple/i}); export const As = createToken({name: "As", pattern: /as/i}); export const If = createToken({name: "If", pattern: /if/i}); export const Start = createToken({name: "Start", pattern: /start(s)?/i}); @@ -53,7 +52,7 @@ export const A = createToken({name: "A", pattern: /a(n)?/i }); //, longer_alt: A export const The = createToken({name: "The", pattern: /the/i }); //, longer_alt: Then}); export const Exactly = createToken({name: "Exactly", pattern: /exact(ly)?/i}); export const Inclusive = createToken({name: "Inclusive", pattern: /inclusive(ly)?/i}); -export const exclusive = createToken({name: "Exclusive", pattern: /exclusive(ly)?/i}); +export const Exclusive = createToken({name: "Exclusive", pattern: /exclusive(ly)?/i}); export const From = createToken({name: "From", pattern: /from/i}); export const To = createToken({name: "To", pattern: /(to|\-|\.\.|\.\.\.)/i}); export const Create = createToken({name: "Create", pattern: /create(s)?/i}); @@ -62,20 +61,23 @@ export const Repeat = createToken({name: "Repeat", pattern: /repeat(s|ing)?/i}); export const Newline = createToken({name: "Newline", pattern: /(new line|newline)/i}); export const None = createToken({name: "None", pattern: /none/i}); export const Neither = createToken({name: "Neither", pattern: /neither/i}); -export const CarriageReturn = createToken({name: "CarriageReturn", pattern: /carriage return/i}); -export const CaseInsensitive = createToken({name: "CaseInsensitive", pattern: /case insensitive/i}); -export const CaseSensitive = createToken({name: "CaseSensitive", pattern: /case sensitive/i}); -export const OrMore = createToken({name: "OrMore", pattern: /\+/ }); +export const CarriageReturn = createToken({name: "Carriage Return", pattern: /carriage return/i}); +export const CaseInsensitive = createToken({name: "Case Insensitive", pattern: /case insensitive/i}); +export const CaseSensitive = createToken({name: "Case Sensitive", pattern: /case sensitive/i}); +export const OrMore = createToken({name: "Or More", pattern: /\+/ }); + +export const LBracket = createToken({name: "Left Bracket", pattern: /\(/ }); +export const RBracket = createToken({name: "Right Bracket", pattern: /\)/ }); export const Indent = createToken({name: "Indent", pattern: /(( ){4}\t)/ }); export const EndOfLine = createToken({name: "EOL", pattern: /\n/ }); -export const WhiteSpace = createToken({name: "WhiteSpace", pattern: /\s+/, group: Lexer.SKIPPED }); -export const SingleLineComment = createToken({name: "SingleLineComment", pattern: /(#|\/\/).*/, group: Lexer.SKIPPED }); -export const MultilineComment = createToken({name: "MultiLineComment", pattern: /\/\*(.*)\*\//, line_breaks: true, group: Lexer.SKIPPED }); +export const WhiteSpace = createToken({name: "Whitespace", pattern: /\s+/, group: Lexer.SKIPPED }); +export const SingleLineComment = createToken({name: "Single-Line Comment", pattern: /(#|\/\/).*/, group: Lexer.SKIPPED }); +export const MultilineComment = createToken({name: "Multi-Line Comment", pattern: /\/\*(.*)\*\//, line_breaks: true, group: Lexer.SKIPPED }); export const Identifier = createToken({name: "Identifier", pattern: /[a-z]\w*/i }); -export const NumberLiteral = createToken({name: "NumberLiteral", pattern: /-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?/ }); -export const StringLiteral = createToken({name: "StringLiteral", pattern: /"(?:[^\\"]|\\(?:[bfnrtv"\\/]|u[0-9a-f]{4}|U[0-9a-f]{8}))*"/i }); +export const NumberLiteral = createToken({name: "Number Literal", pattern: /-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?/ }); +export const StringLiteral = createToken({name: "String Literal", pattern: /"(?:[^\\"]|\\(?:[bfnrtv"\\/]|u[0-9a-f]{4}|U[0-9a-f]{8}))*"/i }); export const AllTokens = [ @@ -103,7 +105,6 @@ export const AllTokens = [ Character, Whitespace, Number, - Multiple, As, If, Start, @@ -129,7 +130,7 @@ export const AllTokens = [ The, Exactly, Inclusive, - exclusive, + Exclusive, From, Create, Called, diff --git a/src/tokens.ts b/src/tokens.ts deleted file mode 100644 index 8aee1da..0000000 --- a/src/tokens.ts +++ /dev/null @@ -1,84 +0,0 @@ -export enum TokenType { - END_OF_STATEMENT, - INDENT, - BETWEEN, - QUOTE, - NUMBER, - PARTIAL_KEYWORD, - KEYWORD_BETWEEN, - KEYWORD_OPTIONAL, - KEYWORD_MATCH, - KEYWORD_THEN, - KEYWORD_AND, - KEYWORD_OR, - KEYWORD_ANY, - KEYWORD_OF, - KEYWORD_NONE, - KEYWORD_NEITHER, - KEYWODE_WORD_SPECIFIER, - KEYWORD_DIGIT_SPECIFIER, - KEYWORD_CHAR_SPECIFIER, - KEYWORD_WHITESPACE_SPECIFIER, - KEYWORD_NUMBER_SPECIFIER, - KEYWORD_MULTIPLE, - KEYWORD_AS, - KEYWORD_IF, - KEYWORD_STARTS, - KEYWORD_WITH, - KEYWORD_ENDS, - KEYWORD_ELSE, - KEYWORD_UNLESS, - KEYWORD_WHILE, - KEYWORD_MORE, - KEYWORD_USING, - KEYWORD_GLOBAL, - KEYWORD_MULTILINE, - KEYWORD_EXACT, - KEYWORD_MATCHING, - KEYWORD_NOT, - KEYWORD_TAB, - KEYWORD_LINEFEED, - KEYWORD_CARRIAGE_RETURN, - KEYWORD_GROUP, - KEYWORD_BY, - KEYWORD_ARTICLE, - KEYWORD_EXACTLY, - KEYWORD_INCLUSIVE, - KEYWORD_EXCLUSIVE, - KEYWORD_FROM, - KEYWORD_TO, - KEYWORD_CREATE, - KEYWORD_CALLED, - KEYWORD_REPEAT, - KEYWORD_NEWLINE, - KEYWORD_CASE_SENSITIVE, - KEYWORD_CASE_INSENSITIVE -} - -export class TokenError extends Error { - constructor(message: string, public line: number, public position: number) { - super(message); - } - - public to_string(): string { - return `Token Error: ${this.line}:${this.position} ${this.message}`; - } -} - -export class Token { - constructor(public type: TokenType, public line: number, public position: number, public length: number, public token_string?: string) { - /* nothing required */ - } - - public to_string(): string { - let str = `${this.line}:${this.position} ${TokenType[this.type]}`; - - if (this.token_string) { - str += ` "${this.token_string}"`; - } - - str += ` (size: ${this.length})`; - - return str; - } -} \ No newline at end of file diff --git a/webpack.config.js b/webpack.full.config.js similarity index 93% rename from webpack.config.js rename to webpack.full.config.js index f3cb4f5..efbc8d2 100644 --- a/webpack.config.js +++ b/webpack.full.config.js @@ -9,6 +9,7 @@ const CopyPlugin = require("copy-webpack-plugin"); const MiniCssExtractPlugin = require("mini-css-extract-plugin"); const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin"); const WebpackBeforeBuildPlugin = require("before-build-webpack"); +const TerserPlugin = require("terser-webpack-plugin"); const config = { prod: true, @@ -82,7 +83,8 @@ module.exports = { ] }, optimization: { - minimizer: [ new OptimizeCSSAssetsPlugin({}) ] + minimize: config.prod, + minimizer: [ new TerserPlugin({cache: true, parallel: true}), new OptimizeCSSAssetsPlugin({}) ] }, plugins: [ new CopyPlugin({ @@ -101,6 +103,6 @@ module.exports = { }, output: { filename: "bundle.min.js", - path: path.resolve("docs") + path: path.resolve(config.dst) } }; \ No newline at end of file diff --git a/webpack.partial.config.js b/webpack.partial.config.js new file mode 100644 index 0000000..93cf5c0 --- /dev/null +++ b/webpack.partial.config.js @@ -0,0 +1,32 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable no-undef */ +const path = require("path"); +const MiniCssExtractPlugin = require("mini-css-extract-plugin"); + +module.exports = { + mode: "development", + entry: path.resolve("./src/", "script.ts"), + module: { + rules: [ + { + test: /\.ts$/, + use: "ts-loader", + exclude: /node_modules/ + }, + { + test: /\.css$/, + use: [ MiniCssExtractPlugin.loader, "css-loader" ] + } + ] + }, + plugins: [ + new MiniCssExtractPlugin({ filename: "bundle.min.css" }), + ], + resolve: { + extensions: [ ".ts", ".js" ] + }, + output: { + filename: "bundle.min.js", + path: path.resolve("docs") + } +}; \ No newline at end of file