Last active
June 10, 2023 20:46
-
-
Save rsoutar/9cf730f1bea9da0eda99c5288b825f4e to your computer and use it in GitHub Desktop.
Simple lexer
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| type TokenType = | |
| | "NUMBER" | |
| | "PLUS" | |
| | "MINUS" | |
| | "MULTIPLY" | |
| | "DIVIDE" | |
| | "LPAREN" | |
| | "RPAREN" | |
| | "EOF"; | |
| interface Token { | |
| type: TokenType; | |
| value: string; | |
| } | |
| class Lexer { | |
| private cursor: number = 0; | |
| constructor(private text: string) {} | |
| private advance(): void { | |
| this.cursor++; | |
| } | |
| private peek(): string { | |
| return this.text[this.cursor] || ""; | |
| } | |
| private makeNumber(): Token { | |
| let numStr = ""; | |
| while (this.peek() && this.peek().match(/[0-9]/)) { | |
| numStr += this.peek(); | |
| this.advance(); | |
| } | |
| return { type: "NUMBER", value: numStr }; | |
| } | |
| public getNextToken(): Token { | |
| while (this.cursor < this.text.length) { | |
| const char = this.peek(); | |
| if (char.match(/\s/)) { | |
| this.advance(); | |
| continue; | |
| } | |
| if (char.match(/[0-9]/)) { | |
| return this.makeNumber(); | |
| } | |
| if (char === "+") { | |
| this.advance(); | |
| return { type: "PLUS", value: char }; | |
| } | |
| if (char === "-") { | |
| this.advance(); | |
| return { type: "MINUS", value: char }; | |
| } | |
| if (char === "*") { | |
| this.advance(); | |
| return { type: "MULTIPLY", value: char }; | |
| } | |
| if (char === "/") { | |
| this.advance(); | |
| return { type: "DIVIDE", value: char }; | |
| } | |
| if (char === "(") { | |
| this.advance(); | |
| return { type: "LPAREN", value: char }; | |
| } | |
| if (char === ")") { | |
| this.advance(); | |
| return { type: "RPAREN", value: char }; | |
| } | |
| throw new Error(`Invalid character: ${char}`); | |
| } | |
| return { type: "EOF", value: "" }; | |
| } | |
| } | |
| let lexer = new Lexer("12 + 34 * (56 - 78)"); | |
| let token: Token; | |
| do { | |
| token = lexer.getNextToken(); | |
| console.log(token); | |
| } while (token.type !== "EOF"); | |
| // Deno is much faster |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment