/******/ (function(modules) { // webpackBootstrap /******/ // The module cache /******/ var installedModules = {}; /******/ /******/ // The require function /******/ function __webpack_require__(moduleId) { /******/ /******/ // Check if module is in cache /******/ if(installedModules[moduleId]) { /******/ return installedModules[moduleId].exports; /******/ } /******/ // Create a new module (and put it into the cache) /******/ var module = installedModules[moduleId] = { /******/ i: moduleId, /******/ l: false, /******/ exports: {} /******/ }; /******/ /******/ // Execute the module function /******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); /******/ /******/ // Flag the module as loaded /******/ module.l = true; /******/ /******/ // Return the exports of the module /******/ return module.exports; /******/ } /******/ /******/ /******/ // expose the modules object (__webpack_modules__) /******/ __webpack_require__.m = modules; /******/ /******/ // expose the module cache /******/ __webpack_require__.c = installedModules; /******/ /******/ // define getter function for harmony exports /******/ __webpack_require__.d = function(exports, name, getter) { /******/ if(!__webpack_require__.o(exports, name)) { /******/ Object.defineProperty(exports, name, { enumerable: true, get: getter }); /******/ } /******/ }; /******/ /******/ // define __esModule on exports /******/ __webpack_require__.r = function(exports) { /******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { /******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); /******/ } /******/ Object.defineProperty(exports, '__esModule', { value: true }); /******/ }; /******/ /******/ // create a fake namespace object /******/ // mode & 1: value is a module id, require it /******/ // mode & 2: merge all properties of value into the ns /******/ // mode & 4: return value when already ns object /******/ // mode & 8|1: behave like require /******/ __webpack_require__.t = function(value, mode) { /******/ if(mode & 1) value = __webpack_require__(value); /******/ if(mode & 8) return value; /******/ if((mode & 4) && typeof value === 'object' && value && value.__esModule) return value; /******/ var ns = Object.create(null); /******/ __webpack_require__.r(ns); /******/ Object.defineProperty(ns, 'default', { enumerable: true, value: value }); /******/ if(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key)); /******/ return ns; /******/ }; /******/ /******/ // getDefaultExport function for compatibility with non-harmony modules /******/ __webpack_require__.n = function(module) { /******/ var getter = module && module.__esModule ? /******/ function getDefault() { return module['default']; } : /******/ function getModuleExports() { return module; }; /******/ __webpack_require__.d(getter, 'a', getter); /******/ return getter; /******/ }; /******/ /******/ // Object.prototype.hasOwnProperty.call /******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); }; /******/ /******/ // __webpack_public_path__ /******/ __webpack_require__.p = ""; /******/ /******/ /******/ // Load entry module and return exports /******/ return __webpack_require__(__webpack_require__.s = 0); /******/ }) /************************************************************************/ /******/ ([ /* 0 */ /***/ (function(module, exports, __webpack_require__) { "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); __webpack_require__(1); const tokenizer_1 = __webpack_require__(2); /* $(function() { }); */ const opts = new tokenizer_1.TokenizerOptions(); const res = tokenizer_1.tokenize("match 1+ thing from thingy", opts); console.log(res); /***/ }), /* 1 */ /***/ (function(module, __webpack_exports__, __webpack_require__) { "use strict"; __webpack_require__.r(__webpack_exports__); // extracted by mini-css-extract-plugin /***/ }), /* 2 */ /***/ (function(module, exports, __webpack_require__) { "use strict"; /*! Copyright (c) 2020 Patrick Demian; Licensed under MIT */ Object.defineProperty(exports, "__esModule", { value: true }); exports.tokenize = exports.TokenizerOptions = void 0; // TODO: replace every version of switch() with switch(.charCodeAt(0)) const tokens_1 = __webpack_require__(3); const keywords = { "optional": tokens_1.TokenType.KEYWORD_OPTIONAL, "optionally": tokens_1.TokenType.KEYWORD_OPTIONAL, "match": tokens_1.TokenType.KEYWORD_MATCH, "then": tokens_1.TokenType.KEYWORD_THEN, "any": tokens_1.TokenType.KEYWORD_ANY, "anything": tokens_1.TokenType.KEYWORD_ANY, "of": tokens_1.TokenType.KEYWORD_OF, "or": tokens_1.TokenType.KEYWORD_OR, "and": tokens_1.TokenType.KEYWORD_AND, "word": tokens_1.TokenType.KEYWODE_WORD_SPECIFIER, "digit": tokens_1.TokenType.KEYWORD_DIGIT_SPECIFIER, "character": tokens_1.TokenType.KEYWORD_CHAR_SPECIFIER, "whitespace": tokens_1.TokenType.KEYWORD_WHITESPACE_SPECIFIER, "number": tokens_1.TokenType.KEYWORD_NUMBER_SPECIFIER, "multiple": tokens_1.TokenType.KEYWORD_MULTIPLE, "as": tokens_1.TokenType.KEYWORD_AS, "if": tokens_1.TokenType.KEYWORD_IF, "starts": tokens_1.TokenType.KEYWORD_STARTS, "with": tokens_1.TokenType.KEYWORD_WITH, "ends": tokens_1.TokenType.KEYWORD_ENDS, "otherwise": tokens_1.TokenType.KEYWORD_ELSE, "else": tokens_1.TokenType.KEYWORD_ELSE, "unless": tokens_1.TokenType.KEYWORD_UNLESS, "while": tokens_1.TokenType.KEYWORD_WHILE, "more": tokens_1.TokenType.KEYWORD_MORE, "using": tokens_1.TokenType.KEYWORD_USING, "global": tokens_1.TokenType.KEYWORD_GLOBAL, "multiline": tokens_1.TokenType.KEYWORD_MULTILINE, "exact": tokens_1.TokenType.KEYWORD_EXACT, "matching": tokens_1.TokenType.KEYWORD_MATCHING, "not": tokens_1.TokenType.KEYWORD_NOT, "between": tokens_1.TokenType.KEYWORD_BETWEEN, "tab": tokens_1.TokenType.KEYWORD_TAB, "linefeed": tokens_1.TokenType.KEYWORD_LINEFEED, "carriage": tokens_1.TokenType.KEYWORD_CARRIAGE, "return": tokens_1.TokenType.KEYWORD_RETURN, "group": tokens_1.TokenType.KEYWORD_GROUP, "by": tokens_1.TokenType.KEYWORD_BY, "an": tokens_1.TokenType.KEYWORD_ARTICLE, "a": tokens_1.TokenType.KEYWORD_ARTICLE, "the": tokens_1.TokenType.KEYWORD_ARTICLE, "exactly": tokens_1.TokenType.KEYWORD_EXACTLY, "inclusive": tokens_1.TokenType.KEYWORD_INCLUSIVE, "inclusively": tokens_1.TokenType.KEYWORD_INCLUSIVE, "exclusive": tokens_1.TokenType.KEYWORD_EXCLUSIVE, "exclusively": tokens_1.TokenType.KEYWORD_EXCLUSIVE, "from": tokens_1.TokenType.KEYWORD_FROM, "to": tokens_1.TokenType.KEYWORD_TO }; const escape_sequences = { "a": "\a", "b": "\b", "e": "\e", "f": "\f", "n": "\n", "r": "\r", "t": "\t", "'": "'", "\"": '"', "\\": "\\", }; class TokenizerOptions { constructor() { this.convert_spaces_to_tabs = false; } } exports.TokenizerOptions = TokenizerOptions; const escape_sequence_hex_regex = new RegExp(/[0-9A-Fa-f]/g); function escape_sequence_gather_hex(input, i, max) { let hex = ""; for (i++; i < input.length && max-- > 0; i++) { if (escape_sequence_hex_regex.test(input[i])) { hex += input[i]; } } return hex; } function escape_sequence_mapper(input, i) { if (escape_sequences[input[i]]) { return { code: escape_sequences[input[i]], read: 1 }; } //variable hex code else if (input[i] === "x") { const hex = escape_sequence_gather_hex(input, ++i, 4); return { code: String.fromCharCode(parseInt(hex, 16)), read: hex.length + 1 }; } //4 hex unicode else if (input[i] === "u") { const unicode = escape_sequence_gather_hex(input, ++i, 4); if (unicode.length !== 4) { return { code: "", read: unicode.length + 1, error: new Error("Bad escape sequence") }; } else { return { code: String.fromCharCode(parseInt(unicode, 16)), read: 5 }; } } else if (input[i] === "U") { const unicode = escape_sequence_gather_hex(input, ++i, 8); if (unicode.length !== 8) { return { code: "", read: unicode.length + 1, error: new Error("Bad escape sequence") }; } else { return { code: String.fromCharCode(parseInt(unicode, 16)), read: 9 }; } } else { // should throw an exception, but gonna just ignore it return { code: input[i], read: 1 }; } } function is_digit(input) { //return /[0-9]/g.test(input); const value = input.charCodeAt(0); return value >= 48 && value <= 57; } function is_char(input) { //return input.toUpperCase() != input.toLowerCase(); //return /[a-zA-Z]/g.test(input); const value = input.charCodeAt(0); return ((value >= 65 && value <= 90) || (value >= 97 && value <= 122)); } /* Basic Tokenizer */ function tokenize(input, options) { let line = 1; let position = 1; const tokens = []; const errors = []; for (let i = 0; i < input.length; i++, position++) { // 4 spaces = 1 tab. That is final. Debate over if (options.convert_spaces_to_tabs && input.startsWith(" ", i)) { tokens.push(new tokens_1.Token(tokens_1.TokenType.INDENT, line, position)); i += 3; position += 3; } // between (ex: 0...3 or 0-3) else if (input.startsWith("...", i)) { tokens.push(new tokens_1.Token(tokens_1.TokenType.BETWEEN, line, position)); i += 2; position += 2; } else if (input.startsWith("..", i)) { tokens.push(new tokens_1.Token(tokens_1.TokenType.BETWEEN, line, position)); i++; position++; } // comments else if (input.startsWith("//", i)) { for (i++, position++; i < input.length; i++, position++) { if (input[i] === "\n") { tokens.push(new tokens_1.Token(tokens_1.TokenType.END_OF_STATEMENT, line, position)); break; } } line++; position = 0; } else if (input.startsWith("/*", i)) { for (i++, position++; i < input.length - 1; i++, position++) { if (input[i] === "*" && input[i + 1] === "/") { tokens.push(new tokens_1.Token(tokens_1.TokenType.END_OF_STATEMENT, line, position)); i++; position++; break; } if (input[i] === "\n") { line++; position = 0; } } if (i === input.length - 1) { errors.push(new tokens_1.TokenError("Unexpected EOF", line, position)); } else { line++; position = 0; } } else if (input.startsWith("\r\n", i)) { tokens.push(new tokens_1.Token(tokens_1.TokenType.END_OF_STATEMENT, line, position)); i++; line++; position = 0; } else { switch (input[i]) { // comment case "#": for (i++, position++; i < input.length; i++, position++) { if (input[i] === "\n") { tokens.push(new tokens_1.Token(tokens_1.TokenType.END_OF_STATEMENT, line, position)); line++; position = 0; break; } } break; // quote case '"': case '\"': { // build up a word between quotes const quote_begin = { line: line, position: position }; const quote_char = input[i]; let found_ending = false; let quote = ""; do { i++; position++; if (input[i] === "\\") { i++; position++; const sequence = escape_sequence_mapper(input, i); if (sequence.error) { errors.push(new tokens_1.TokenError(sequence.error.message, line, position)); } position += sequence.read; i += sequence.read; quote += sequence.code; } else if (input[i] === quote_char) { found_ending = true; break; } else if (input[i] === "\n") { line++; position = 0; break; } else { quote += input[i]; } } while (i < input.length); if (found_ending) { tokens.push(new tokens_1.Token(tokens_1.TokenType.QUOTE, line, position, quote)); } else { //we reached the end of the line or the end of the file errors.push(new tokens_1.TokenError(`Unexpected end of quote. Quote began at ${quote_begin.line}:${quote_begin.position}`, line, position)); line++; position = 0; } break; } // between (ex: 0...3 or 0-3) case "-": tokens.push(new tokens_1.Token(tokens_1.TokenType.BETWEEN, line, position)); break; case "\n": tokens.push(new tokens_1.Token(tokens_1.TokenType.END_OF_STATEMENT, line, position)); break; case "\r": // ignore break; case "\t": tokens.push(new tokens_1.Token(tokens_1.TokenType.INDENT, line, position)); break; case " ": break; default: // is digit? build up a number if (is_digit(input[i])) { let digits = input[i]; do { i++; position++; digits += input[i]; } while (i + 1 < input.length && is_digit(input[i + 1])); tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, digits)); } // is char? build up a word else if (is_char(input[i])) { let text = input[i]; do { i++; position++; text += input[i]; } while (i + 1 < input.length && is_char(input[i + 1])); const keyword_text = text.toLowerCase(); if (keywords[keyword_text]) { tokens.push(new tokens_1.Token(keywords[keyword_text], line, position)); } else { switch (keyword_text) { case "none": case "zero": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "0")); break; case "one": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "1")); break; case "two": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "2")); break; case "three": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "3")); break; case "four": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "4")); break; case "five": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "5")); break; case "six": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "6")); break; case "seven": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "7")); break; case "eight": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "8")); break; case "nine": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "9")); break; case "ten": tokens.push(new tokens_1.Token(tokens_1.TokenType.NUMBER, line, position, "10")); break; default: errors.push(new tokens_1.TokenError(`Unknown keyword ${text}`, line, position)); break; } } } else { errors.push(new tokens_1.TokenError(`Unknown character in text: ${input.charCodeAt(i)}`, line, position)); } break; } } } return { tokens: tokens, errors: errors }; } exports.tokenize = tokenize; /***/ }), /* 3 */ /***/ (function(module, exports, __webpack_require__) { "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.Token = exports.TokenError = exports.TokenType = void 0; var TokenType; (function (TokenType) { TokenType[TokenType["END_OF_STATEMENT"] = 0] = "END_OF_STATEMENT"; TokenType[TokenType["INDENT"] = 1] = "INDENT"; TokenType[TokenType["BETWEEN"] = 2] = "BETWEEN"; TokenType[TokenType["QUOTE"] = 3] = "QUOTE"; TokenType[TokenType["NUMBER"] = 4] = "NUMBER"; TokenType[TokenType["KEYWORD_BETWEEN"] = 5] = "KEYWORD_BETWEEN"; TokenType[TokenType["KEYWORD_OPTIONAL"] = 6] = "KEYWORD_OPTIONAL"; TokenType[TokenType["KEYWORD_MATCH"] = 7] = "KEYWORD_MATCH"; TokenType[TokenType["KEYWORD_THEN"] = 8] = "KEYWORD_THEN"; TokenType[TokenType["KEYWORD_AND"] = 9] = "KEYWORD_AND"; TokenType[TokenType["KEYWORD_OR"] = 10] = "KEYWORD_OR"; TokenType[TokenType["KEYWORD_ANY"] = 11] = "KEYWORD_ANY"; TokenType[TokenType["KEYWORD_OF"] = 12] = "KEYWORD_OF"; TokenType[TokenType["KEYWODE_WORD_SPECIFIER"] = 13] = "KEYWODE_WORD_SPECIFIER"; TokenType[TokenType["KEYWORD_DIGIT_SPECIFIER"] = 14] = "KEYWORD_DIGIT_SPECIFIER"; TokenType[TokenType["KEYWORD_CHAR_SPECIFIER"] = 15] = "KEYWORD_CHAR_SPECIFIER"; TokenType[TokenType["KEYWORD_WHITESPACE_SPECIFIER"] = 16] = "KEYWORD_WHITESPACE_SPECIFIER"; TokenType[TokenType["KEYWORD_NUMBER_SPECIFIER"] = 17] = "KEYWORD_NUMBER_SPECIFIER"; TokenType[TokenType["KEYWORD_MULTIPLE"] = 18] = "KEYWORD_MULTIPLE"; TokenType[TokenType["KEYWORD_AS"] = 19] = "KEYWORD_AS"; TokenType[TokenType["KEYWORD_IF"] = 20] = "KEYWORD_IF"; TokenType[TokenType["KEYWORD_STARTS"] = 21] = "KEYWORD_STARTS"; TokenType[TokenType["KEYWORD_WITH"] = 22] = "KEYWORD_WITH"; TokenType[TokenType["KEYWORD_ENDS"] = 23] = "KEYWORD_ENDS"; TokenType[TokenType["KEYWORD_ELSE"] = 24] = "KEYWORD_ELSE"; TokenType[TokenType["KEYWORD_UNLESS"] = 25] = "KEYWORD_UNLESS"; TokenType[TokenType["KEYWORD_WHILE"] = 26] = "KEYWORD_WHILE"; TokenType[TokenType["KEYWORD_MORE"] = 27] = "KEYWORD_MORE"; TokenType[TokenType["KEYWORD_USING"] = 28] = "KEYWORD_USING"; TokenType[TokenType["KEYWORD_GLOBAL"] = 29] = "KEYWORD_GLOBAL"; TokenType[TokenType["KEYWORD_MULTILINE"] = 30] = "KEYWORD_MULTILINE"; TokenType[TokenType["KEYWORD_EXACT"] = 31] = "KEYWORD_EXACT"; TokenType[TokenType["KEYWORD_MATCHING"] = 32] = "KEYWORD_MATCHING"; TokenType[TokenType["KEYWORD_NOT"] = 33] = "KEYWORD_NOT"; TokenType[TokenType["KEYWORD_TAB"] = 34] = "KEYWORD_TAB"; TokenType[TokenType["KEYWORD_LINEFEED"] = 35] = "KEYWORD_LINEFEED"; TokenType[TokenType["KEYWORD_CARRIAGE"] = 36] = "KEYWORD_CARRIAGE"; TokenType[TokenType["KEYWORD_RETURN"] = 37] = "KEYWORD_RETURN"; TokenType[TokenType["KEYWORD_GROUP"] = 38] = "KEYWORD_GROUP"; TokenType[TokenType["KEYWORD_BY"] = 39] = "KEYWORD_BY"; TokenType[TokenType["KEYWORD_ARTICLE"] = 40] = "KEYWORD_ARTICLE"; TokenType[TokenType["KEYWORD_EXACTLY"] = 41] = "KEYWORD_EXACTLY"; TokenType[TokenType["KEYWORD_INCLUSIVE"] = 42] = "KEYWORD_INCLUSIVE"; TokenType[TokenType["KEYWORD_EXCLUSIVE"] = 43] = "KEYWORD_EXCLUSIVE"; TokenType[TokenType["KEYWORD_FROM"] = 44] = "KEYWORD_FROM"; TokenType[TokenType["KEYWORD_TO"] = 45] = "KEYWORD_TO"; })(TokenType = exports.TokenType || (exports.TokenType = {})); class TokenError extends Error { constructor(message, line, position) { super(message); this.line = line; this.position = position; } to_string() { return `${this.line}:${this.position} ${this.message}`; } } exports.TokenError = TokenError; class Token { constructor(type, line, position, token_string) { this.type = type; this.line = line; this.position = position; this.token_string = token_string; } } exports.Token = Token; /***/ }) /******/ ]);