baby's first lexer

2026-01-30 23:02:46 -07:00 · 2026-01-30 23:02:46 -07:00 · f74d374555
commit f74d374555
parent 920151f49c
4 changed files with 152 additions and 7 deletions
--- a/src/lexer.ts
+++ b/src/lexer.ts
@ -0,0 +1,139 @@
+export type Token =
+    | { kind: 'let' }
+    | { kind: 'in' }
+
+    | { kind: 'number', value: number }
+    | { kind: 'ident', value: string }
+
+    | { kind: 'equals' }
+
+    | { kind: 'open-paren' }
+    | { kind: 'close-paren' }
+
+    | { kind: 'comma' }
+    | { kind: 'arrow' }
+
+    | { kind: 'if' }
+    | { kind: 'then' }
+    | { kind: 'else' }
+
+    | { kind: 'true' }
+    | { kind: 'false' }
+
+    | { kind: 'plus' }
+    | { kind: 'minus' }
+    | { kind: 'star' }
+    | { kind: 'slash' }
+    | { kind: 'less-than' }
+    | { kind: 'greater-than' }
+    | { kind: 'double-equals' }
+
+export function tokenize(source: string): Token[] {
+    const tokens = [];
+
+    let i = 0;
+    while (i < source.length) {
+        const char = source[i];
+
+        // skip whitespace
+        if (/\s/.test(char)) {
+            i++;
+            continue;
+        }
+
+        // Multi-char: numbers
+        if (/[0-9]/.test(char)) {
+            let num = '';
+            while (i < source.length && /[0-9]/.test(source[i])) {
+                num += source[i];
+                i++;
+            }
+            tokens.push({ kind: 'number', value: parseInt(num) });
+            continue;
+        }
+
+        // Multi-char: equals
+        if (char === '=') {
+            const nextChar = source[i + 1];
+
+            if (nextChar === '=') {
+                tokens.push({ kind: 'double-equals' });
+                i++;
+                continue;
+            } else if (nextChar === '>') {
+                tokens.push({ kind: 'arrow' });
+                i++;
+                continue;
+            } else {
+                tokens.push({ kind: 'equals' });
+                i++;
+                continue;
+            }
+        }
+
+        // Multi-char: strings
+        if (/[A-Za-z]/.test(char)) {
+            let str = '';
+            while (i < source.length && /[A-Za-z]/.test(source[i])) {
+                str += source[i];
+                i++;
+            }
+
+            if (str === 'let') {
+                tokens.push({ kind: 'let' });
+            } else if (str === 'in') {
+                tokens.push({ kind: 'in' });
+            } else if (str === 'if') {
+                tokens.push({ kind: 'if' });
+            } else if (str === 'then') {
+                tokens.push({ kind: 'then' });
+            } else if (str === 'else') {
+                tokens.push({ kind: 'else' });
+            } else if (str === 'true') {
+                tokens.push({ kind: 'true' });
+            } else if (str === 'false') {
+                tokens.push({ kind: 'false' });
+            } else {
+                tokens.push({ kind: 'ident', value: str });
+            }
+
+            continue;
+        }
+
+        // TODO: floats
+
+        switch (char) {
+            case ',':
+                tokens.push({ kind: 'comma' });
+                break;
+            case '+':
+                tokens.push({ kind: 'plus' });
+                break;
+            case '-':
+                tokens.push({ kind: 'minus' });
+                break;
+            case '*':
+                tokens.push({ kind: 'star' });
+                break;
+            case '/':
+                tokens.push({ kind: 'slash' });
+                break;
+            case '(':
+                tokens.push({ kind: 'open-paren' });
+                break;
+            case ')':
+                tokens.push({ kind: 'close-paren' });
+                break;
+            case '<':
+                tokens.push({ kind: 'less-than' });
+                break;
+            case '>':
+                tokens.push({ kind: 'greater-than' });
+                break;
+        }
+
+        i++;
+    }
+
+    return tokens;
+}