baby's first lexer

src/ast.ts

@@ -1,4 +1,4 @@
-import type { Value, Closure } from './types';
+import type { Value } from './types';
 
 export type Literal = {
     kind: 'literal'

src/interpreter.ts

@@ -1,6 +1,6 @@
 import type { AST } from './ast';
 import type { Env } from './env';
-import type { Value, Closure } from './types';
+import type { Value } from './types';
 
 export function evaluate(ast: AST, env: Env): Value {
     switch (ast.kind) {
@@ -24,12 +24,12 @@ export function evaluate(ast: AST, env: Env): Value {
             return evaluate(ast.body, newEnv);
 
         case 'if':
-            const { kind, value } = evaluate(ast.condition, env);
+            const res = evaluate(ast.condition, env);
 
-            if (kind !== 'bool')
+            if (res.kind !== 'bool')
                 throw new Error('Condition must be bool');
 
-            if (value) {
+            if (res.value) {
                 return evaluate(ast.then, env);
             } else {
                 return evaluate(ast.else, env);
@@ -70,13 +70,16 @@ export function evaluate(ast: AST, env: Env): Value {
 }
 
 function evaluateBinaryOp(op: string, left: Value, right: Value): Value {
-    const { value: leftValue, kind: leftKind } = left;
-    const { value: rightValue, kind: rightKind } = right;
+    const leftKind = left.kind;
+    const rightKind = right.kind;
 
     const bothNumbers = ((leftKind === 'int' || leftKind === 'float')) && ((rightKind === 'int' || rightKind === 'float'));
     if (!bothNumbers)
         throw new Error(`Not numbers: ${left}, ${right}`);
 
+    const leftValue = left.value;
+    const rightValue = right.value;
+
     switch (op) {
         case '+':
             return { value: leftValue + rightValue, kind: 'int' };

src/lexer.ts

@@ -0,0 +1,139 @@
+export type Token =
+    | { kind: 'let' }
+    | { kind: 'in' }
+
+    | { kind: 'number', value: number }
+    | { kind: 'ident', value: string }
+
+    | { kind: 'equals' }
+
+    | { kind: 'open-paren' }
+    | { kind: 'close-paren' }
+
+    | { kind: 'comma' }
+    | { kind: 'arrow' }
+
+    | { kind: 'if' }
+    | { kind: 'then' }
+    | { kind: 'else' }
+
+    | { kind: 'true' }
+    | { kind: 'false' }
+
+    | { kind: 'plus' }
+    | { kind: 'minus' }
+    | { kind: 'star' }
+    | { kind: 'slash' }
+    | { kind: 'less-than' }
+    | { kind: 'greater-than' }
+    | { kind: 'double-equals' }
+
+export function tokenize(source: string): Token[] {
+    const tokens = [];
+
+    let i = 0;
+    while (i < source.length) {
+        const char = source[i];
+
+        // skip whitespace
+        if (/\s/.test(char)) {
+            i++;
+            continue;
+        }
+
+        // Multi-char: numbers
+        if (/[0-9]/.test(char)) {
+            let num = '';
+            while (i < source.length && /[0-9]/.test(source[i])) {
+                num += source[i];
+                i++;
+            }
+            tokens.push({ kind: 'number', value: parseInt(num) });
+            continue;
+        }
+
+        // Multi-char: equals
+        if (char === '=') {
+            const nextChar = source[i + 1];
+
+            if (nextChar === '=') {
+                tokens.push({ kind: 'double-equals' });
+                i++;
+                continue;
+            } else if (nextChar === '>') {
+                tokens.push({ kind: 'arrow' });
+                i++;
+                continue;
+            } else {
+                tokens.push({ kind: 'equals' });
+                i++;
+                continue;
+            }
+        }
+
+        // Multi-char: strings
+        if (/[A-Za-z]/.test(char)) {
+            let str = '';
+            while (i < source.length && /[A-Za-z]/.test(source[i])) {
+                str += source[i];
+                i++;
+            }
+
+            if (str === 'let') {
+                tokens.push({ kind: 'let' });
+            } else if (str === 'in') {
+                tokens.push({ kind: 'in' });
+            } else if (str === 'if') {
+                tokens.push({ kind: 'if' });
+            } else if (str === 'then') {
+                tokens.push({ kind: 'then' });
+            } else if (str === 'else') {
+                tokens.push({ kind: 'else' });
+            } else if (str === 'true') {
+                tokens.push({ kind: 'true' });
+            } else if (str === 'false') {
+                tokens.push({ kind: 'false' });
+            } else {
+                tokens.push({ kind: 'ident', value: str });
+            }
+
+            continue;
+        }
+
+        // TODO: floats
+
+        switch (char) {
+            case ',':
+                tokens.push({ kind: 'comma' });
+                break;
+            case '+':
+                tokens.push({ kind: 'plus' });
+                break;
+            case '-':
+                tokens.push({ kind: 'minus' });
+                break;
+            case '*':
+                tokens.push({ kind: 'star' });
+                break;
+            case '/':
+                tokens.push({ kind: 'slash' });
+                break;
+            case '(':
+                tokens.push({ kind: 'open-paren' });
+                break;
+            case ')':
+                tokens.push({ kind: 'close-paren' });
+                break;
+            case '<':
+                tokens.push({ kind: 'less-than' });
+                break;
+            case '>':
+                tokens.push({ kind: 'greater-than' });
+                break;
+        }
+
+        i++;
+    }
+
+    return tokens;
+}

src/main.ts

@@ -1,6 +1,7 @@
 import { evaluate } from './interpreter'
 import type { AST } from './ast'
 import type { Env } from './env'
+import { tokenize } from './lexer'
 
 const ast: AST = {
     kind: 'binaryop',
@@ -72,3 +73,5 @@ const env2: Env = new Map();
 const res2 = evaluate(ast2, env2);
 
 console.log(res2);
+
+console.log(tokenize("let x = 5"));