/*! Copyright (c) 2021 Patrick Demian; Licensed under MIT */ /** * The Lexer for Human2Regex * @packageDocumentation */ import { Lexer, IToken, createTokenInstance, ILexingError } from "chevrotain"; import { last, findLastIndex, CommonError } from "./utilities"; import { Indent, Outdent, EndOfLine, AllTokens } from "./tokens"; /** * Defines the type of indents the lexer will allow */ export enum IndentType { Tabs, Spaces, Both } /** * The options for the Lexer */ export class Human2RegexLexerOptions { /** * Constructor for the Human2RegexLexerOptions * * @param skip_validations If true, the lexer will skip validations (~25% faster) * @param type The type of indents the lexer will allow * @param spaces_per_tab Number of spaces per tab */ constructor(public skip_validations = false, public type: IndentType = IndentType.Both, public spaces_per_tab: number = 4) { /* empty */ } } /** * Tokenization result */ export class TokenizeResult { /** * Constructor for the TokenizeResult * * @param tokens The token stream * @param errors A list of lexing errors */ public constructor(public tokens: IToken[], public errors: CommonError[]) { /* empty */ } } /** * Human2Regex Lexer * * @remarks Only 1 lexer instance allowed due to a technical limitation and performance reasons */ export class Human2RegexLexer { private static already_init = false; private lexer!: Lexer; private options!: Human2RegexLexerOptions; /** * Human2Regex Lexer * * @remarks Only 1 lexer instance allowed due to a technical limitation and performance reasons * @param options options for the lexer * @see Human2RegexLexerOptions */ constructor(options: Human2RegexLexerOptions = new Human2RegexLexerOptions()) { if (Human2RegexLexer.already_init) { throw new Error("Only 1 instance of Human2RegexLexer allowed"); } Human2RegexLexer.already_init = true; this.setOptions(options); } /** * Sets the options for this lexer * * @param options options for the lexer * @see Human2RegexLexerOptions */ public setOptions(options: Human2RegexLexerOptions): void { this.options = options; let indent_regex: RegExp | null = null; // Generate an index lexer (accepts tabs or spaces or both based on options) if (this.options.type === IndentType.Tabs) { indent_regex = /\t/y; } else { let reg = ` {${this.options.spaces_per_tab}}`; if (this.options.type === IndentType.Both) { reg += "|\\t"; } indent_regex = new RegExp(reg, "y"); } Indent.PATTERN = indent_regex; this.lexer = new Lexer(AllTokens, { ensureOptimizations: true, skipValidations: options.skip_validations }); } private lexError(token: IToken): ILexingError { return { offset: token.startOffset, line: token.startLine ?? NaN, column: token.startColumn ?? NaN, length: token.endOffset ?? NaN - token.startOffset, message: "Unexpected indentation found" }; } /** * Tokenizes the given text * * @param text the text to analyze * @returns a tokenize result which contains the token stream and error list * @public */ public tokenize(text: string): TokenizeResult { const lex_result = this.lexer.tokenize(text); if (lex_result.tokens.length === 0) { return new TokenizeResult(lex_result.tokens, lex_result.errors.map(CommonError.fromLexError)); } const tokens: IToken[] = []; const indent_stack = [ 0 ]; let curr_indent_level = 0; let start_of_line = true; let had_indents = false; // create Outdents for (let i = 0; i < lex_result.tokens.length; i++) { // EoL? check for indents next (by setting startOfLine = true) if (lex_result.tokens[i].tokenType === EndOfLine) { if (tokens.length === 0 || tokens[tokens.length-1].tokenType === EndOfLine) { // Ignore multiple EOLs and ignore first EOL } else { start_of_line = true; tokens.push(lex_result.tokens[i]); } } // start with 1 indent. Append all other indents else if (lex_result.tokens[i].tokenType === Indent) { had_indents = true; curr_indent_level = 1; const start_token = lex_result.tokens[i]; let length = lex_result.tokens[i].image.length; // grab all the indents (and their length) while (lex_result.tokens.length > i+1 && lex_result.tokens[i+1].tokenType === Indent) { curr_indent_level++; i++; length += lex_result.tokens[i].image.length; } start_token.endOffset = start_token.startOffset + length; start_token.endColumn = lex_result.tokens[i].endColumn; // must be the same line //start_token.endLine = lex_result.tokens[i].endLine; // are we an empty line? if (lex_result.tokens.length > i+1 && lex_result.tokens[i+1].tokenType === EndOfLine) { // Ignore all indents AND newline // continue; } // new indent is too far ahead else if (!start_of_line || (curr_indent_level > last(indent_stack) + 1)) { lex_result.errors.push(this.lexError(start_token)); } // new indent is just 1 above else if (curr_indent_level > last(indent_stack)) { indent_stack.push(curr_indent_level); tokens.push(start_token); } // new indent is below the past indent else if (curr_indent_level < last(indent_stack)) { const index = findLastIndex(indent_stack, curr_indent_level); if (index < 0) { lex_result.errors.push(this.lexError(start_token)); } else { const number_of_dedents = indent_stack.length - index - 1; for (let j = 0; j < number_of_dedents; j++) { indent_stack.pop(); tokens.push(createTokenInstance(Outdent, "", start_token.startOffset, start_token.startOffset + length, start_token.startLine ?? NaN, start_token.endLine ?? NaN, start_token.startColumn ?? NaN, (start_token.startColumn ?? NaN) + length)); } } } else { // same indent level: don't care // continue; } } else { if (start_of_line && !had_indents) { const tok = lex_result.tokens[i]; //add remaining Outdents while (indent_stack.length > 1) { indent_stack.pop(); tokens.push(createTokenInstance(Outdent, "", tok.startOffset, tok.startOffset, tok.startLine ?? NaN, NaN, tok.startColumn ?? NaN, NaN)); } } start_of_line = false; had_indents = false; tokens.push(lex_result.tokens[i]); } } const tok = last(tokens); // Do we have an EOL marker at the end? if (tokens.length > 0 && tok.tokenType !== EndOfLine) { tokens.push(createTokenInstance(EndOfLine, "\n", tok.endOffset ?? NaN, tok.endOffset ?? NaN, tok.startLine ?? NaN, NaN, tok.startColumn ?? NaN, NaN)); } //add remaining Outdents while (indent_stack.length > 1) { indent_stack.pop(); tokens.push(createTokenInstance(Outdent, "", tok.endOffset ?? NaN, tok.endOffset ?? NaN, tok.startLine ?? NaN, NaN, tok.startColumn ?? NaN, NaN)); } return new TokenizeResult(tokens, lex_result.errors.map(CommonError.fromLexError)); } }