not stable lol

.gitignore

@@ -1,4 +1,4 @@
-disco
+
 out
 *.o
 node_modules

disco.disco

@@ -1,2 +1,3 @@
 link log
-log("Hello World")
+const a = "a"
+log("hello world")

disco_test.asm

@@ -1,15 +1,19 @@
+bits 64
+default rel
 section .data
-  EFDNYLFZ db 72,101,108,108,111,32,87,111,114,108,100,0
+  GSDGYLUR db 97,0
+  STVGNPWI db 104,101,108,108,111,32,119,111,114,108,100,0
 section .text
-  global _start
-_start:
+  global _main
+_main:
   push rbp
   mov rbp, rsp
-  mov rdi, EFDNYLFZ
+  push qword [rel GSDGYLUR]
+  mov rdi, STVGNPWI
   call _log
   mov rsp, rbp
   pop rbp
-  mov rax, 60
+  mov rax, 0x02000001
   mov rdi, 0
   syscall
 _log:
@@ -24,12 +28,12 @@ _log_loop:
   jmp _log_loop
 _log_loop_end:
   mov rdx, rbx
-  mov rax, 1
+  mov rax, 0x02000004
   mov rdi, 1
   pop rsi
   syscall
   push 10
-  mov rax, 1
+  mov rax, 0x02000004
   mov rdi, 1
   mov rsi, rsp
   mov rdx, 1

src/compiler.ts

@@ -23,7 +23,7 @@ const localVariables = new Map();
 const sections = {
   preamble() {
     if(process.platform === 'darwin') {
-      return '  global _main\n';
+      return 'bits 64\ndefault rel\n';
     } else {
       return '';
     }
@@ -37,10 +37,21 @@ const sections = {
   },
   text() {
     if(process.platform === 'darwin') {
-      return 'section .text\n_main:\n  push rbp\n  mov rbp, rsp\n  '
-      + statements.join('\n  ')
-      + '\n  mov rsp, rbp\n  pop rbp\n  mov rax, 0x02000001\n  mov rdi, 0\n  syscall\n'
-      + [...linkedLibraries.values()].map(({asmName, asm}) => asmName + ':\n' + asm).join('\n')
+      return (
+        'section .text\n' +
+        '  global _main\n' +
+        '_main:\n' +
+        '  push rbp\n' +
+        '  mov rbp, rsp\n' +
+        statements.map(v => `  ${v}\n`).join('') +
+        '  mov rsp, rbp\n' +
+        '  pop rbp\n' +
+        '  mov rax, 0x02000001\n' +
+        '  mov rdi, 0\n' +
+        '  syscall\n' + [...linkedLibraries.values()]
+          .map(({asmName, asm}) => asmName + ':\n' + asm)
+          .join('\n')
+      );
     } else {
       return 'section .text\n  global _start\n_start:\n  push rbp\n  mov rbp, rsp\n  '
         + statements.join('\n  ')
@@ -106,7 +117,10 @@ function compileVariable(name, value) {
   });
   if(value.type === 'string') {
     const variableName = compileStringLiteral(value.value);
-    statements.push('push ' + variableName)
+    if(process.platform === 'darwin')
+      statements.push(`push qword [rel ${variableName}]`);
+    else
+      statements.push('push ' + variableName);
   } else {
     console.error('dont know how to set a variable to a non string lol')
   }

src/disco.ts

@@ -3,8 +3,8 @@
 import { readFileSync } from "fs";
 import { compile } from "./compiler";
 import grammar from "./grammar";
-import { tokenize } from "./tokenizer";
 import colorize from "./util/asm/colorize";
+import tokenize from "./util/disco/tokenizer";
 import { printTokens } from "./util/utils";
 
 console.log();
@@ -32,8 +32,8 @@ const asmFile = compile(ast)
 try {
   console.log();
   console.log('=== ASM ===');
-  console.log(colorize(asmFile));
   require('fs').writeFileSync('disco_test.asm', asmFile);
+  console.log(colorize(asmFile));
 
   console.log();
   console.log('=== nasm ===');
@@ -59,6 +59,7 @@ function ld() {
     require('child_process').execSync([
       'ld', 'disco_test.o',
       '-o', 'disco_test',
+      '-no_pie',
       '-macosx_version_min', '11.0',
       '-L', '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib',
       '-lSystem'

src/earley.ts

@@ -37,6 +37,7 @@ export class Terminal extends Token { static terminal: true = true };
 // these tokens are special, for formatting and generalization reasons.
 export class $Newline extends Terminal { }
 export class $Whitespace extends Terminal { }
+export class $EOF extends Terminal { }
 
 function isTerminal(tokenClass: TokenClass): tokenClass is TerminalTokenClass {
   return tokenClass.terminal;

src/grammar.ts

@@ -1,50 +1,32 @@
-import { Grammar, NonTerminal, Production, Terminal, Token } from "./earley";
+import { $Newline, Grammar, NonTerminal, Production, Terminal, Token } from "./earley";
 import { AST } from './ast';
-
-export class $KeywordLink extends Terminal { }
-export class $KeywordEquals extends Terminal { }
-export class $KeywordLParen extends Terminal { }
-export class $KeywordRParen extends Terminal { }
-export class $KeywordConst extends Terminal { }
-
-export class $String extends Terminal {}
-export class $Identifier extends Terminal {}
-
-export class $Newline extends Terminal { }
-
-export class $Program extends NonTerminal { }
-export class $Statement extends NonTerminal { }
-export class $LinkStatement extends NonTerminal { }
-export class $VariableDeclaration extends NonTerminal { }
-export class $Expression extends NonTerminal { }
-export class $InvocationExpression extends NonTerminal { }
-export class $VariableReference extends NonTerminal { }
+import * as t from './util/disco/tokens';
 
 const ps: Production[] = [
-  { left: $Program, right: [$Statement], resolver: (s) => !!s ? AST.Body([s]) : AST.Body([]) },
-  { left: $Program, right: [$Statement, $Program], resolver: (s, ss) => !!s ? AST.Body([s, ...ss.value]) : ss},
+  { left: t.$Program, right: [t.$Statement], resolver: (s) => !!s ? AST.Body([s]) : AST.Body([]) },
+  { left: t.$Program, right: [t.$Statement, t.$Program], resolver: (s, ss) => !!s ? AST.Body([s, ...ss.value]) : ss},
 
-  { left: $Statement, right: [$Newline], resolver: () => false },
-  { left: $Statement, right: [$LinkStatement], resolver: a => a },
-  { left: $Statement, right: [$VariableDeclaration], resolver: a => a },
-  { left: $Statement, right: [$Expression], resolver: a => a },
+  { left: t.$Statement, right: [$Newline], resolver: () => false },
+  { left: t.$Statement, right: [t.$LinkStatement], resolver: a => a },
+  { left: t.$Statement, right: [t.$VariableDeclaration], resolver: a => a },
+  { left: t.$Statement, right: [t.$Expression], resolver: a => a },
 
-  { left: $Expression, right: [$String], resolver: (s: $String) => AST.String(s.value) },
-  { left: $Expression, right: [$InvocationExpression], resolver: a => a },
-  { left: $Expression, right: [$VariableReference], resolver: a => a },
+  { left: t.$Expression, right: [t.$String], resolver: (s: t.$String) => AST.String(s.value) },
+  { left: t.$Expression, right: [t.$InvocationExpression], resolver: a => a },
+  { left: t.$Expression, right: [t.$VariableReference], resolver: a => a },
 
-  { left: $VariableReference, right: [$Identifier], resolver: (identifier: $Identifier) => AST.VariableReference(identifier.value) },
+  { left: t.$VariableReference, right: [t.$Identifier], resolver: (identifier: t.$Identifier) => AST.VariableReference(identifier.value) },
 
-  { left: $InvocationExpression, right: [$Identifier, $KeywordLParen, $Expression, $KeywordRParen],
-    resolver: (identifier: $Identifier, _, arg: any, __) => AST.Invocation(identifier.value, arg) },
+  { left: t.$InvocationExpression, right: [t.$Identifier, t.$KeywordLParen, t.$Expression, t.$KeywordRParen],
+    resolver: (identifier: t.$Identifier, _, arg: any, __) => AST.Invocation(identifier.value, arg) },
 
-  { left: $VariableDeclaration, right: [$KeywordConst, $Identifier, $KeywordEquals, $Expression],
-    resolver: (_, identifier: $Identifier, __, value: any) => AST.Const(identifier.value, value) },
+  { left: t.$VariableDeclaration, right: [t.$KeywordConst, t.$Identifier, t.$KeywordEquals, t.$Expression],
+    resolver: (_, identifier: t.$Identifier, __, value: any) => AST.Const(identifier.value, value) },
 
-  { left: $LinkStatement, right: [$KeywordLink, $Identifier], resolver: (_, identifier: $Identifier) => AST.Link(identifier.value) },
+  { left: t.$LinkStatement, right: [t.$KeywordLink, t.$Identifier], resolver: (_, identifier: t.$Identifier) => AST.Link(identifier.value) },
 
 ]
 
-const grammar = new Grammar(ps, $Program);
+const grammar = new Grammar(ps, t.$Program);
 
 export default grammar;

src/tokenizer.ts

@@ -1,85 +0,0 @@
-import * as chalk from 'chalk';
-import { readFileSync, writeFileSync } from 'fs';
-import { $Identifier, $KeywordConst, $KeywordEquals, $KeywordLink, $KeywordLParen, $KeywordRParen, $Newline, $String } from './grammar';
-
-
-const keywords = new Map([
-  ['=', $KeywordEquals],
-  ['(', $KeywordLParen],
-  [')', $KeywordRParen],
-  ['link', $KeywordLink],
-  ['const', $KeywordConst],
-]);
-
-export function tokenize(string) {
-  let inString = false;
-  let escaping = false;
-  let tokens = [];
-  let token = '';
-  // let line = 1;
-  // let col = 1;
-  // const newline = () => (col = 1, line ++);
-  // const nextColumn = () => line ++;
-  const resetToken = () => token = '';
-  const addToken = (_token?) => {
-    if(_token) {
-      token = _token;
-    }
-    if(token.trim() !== '') {
-      if(keywords.has(token)) {
-        const kwTokenClass = keywords.get(token);
-        tokens.push(new kwTokenClass(0, 0, token));
-      } else if (isStringDelim(token[0]))
-        tokens.push(new $String(0, 0, token.substring(1, token.length - 1)));
-      else if (token === 'NEWLINE')
-        tokens.push(new $Newline(0, 0, token))
-      else
-        tokens.push(new $Identifier(0, 0, token));
-      resetToken();
-    }
-  }
-  // let _line = line;
-  // let _col = col;
-
-  const isWhitespace = (char) => [' ', '\n', '\t', '\r'].includes(char);
-  const isNewline = (char) => char === '\n';
-  const isSingleCharToken = (char) => ['(', ')', '='].includes(char);
-  const isStringDelim = (char) => ["'", '"'].includes(char);
-  const isEscapeChar = (char) => char === '\\';
-  const escape = (char) => (char === 'n' ? '\n'
-                          : char === 't' ? '\t'
-                          : char === 'r' ? '\r' : char)
-
-  for (const char of string) {
-    if(isNewline(char)) {
-      // newline();
-      addToken();
-      // only add newlines if we've actually started tokens...
-      if(tokens.length > 0)
-        addToken('NEWLINE')
-    } else if (escaping) {
-      token += escape(char)
-      escaping = false;
-    } else if (isStringDelim(char)) {
-      token += char;
-      inString = !inString;
-    } else if (inString) {
-      if(isEscapeChar(char)) {
-        escaping = true;
-      } else {
-        token += char
-      }
-    } else if(isSingleCharToken(char)) {
-      addToken();
-      addToken(char);
-    } else if(isWhitespace(char)) {
-      addToken();
-    } else {
-      token += char;
-    }
-    // if(!isNewline(char))
-    //   nextColumn();
-  }
-
-  return tokens;
-}

src/util/asm/grammar.ts

@@ -26,8 +26,12 @@ export default new Grammar([
   { left: t.$Line, right: [t.$Global, t.$Identifier],
     resolver: (_, {value}) => `  ${ansi(...keywordColor).bold('global')} ${ansi(...identifierColor)(value)}` },
   { left: t.$Line, right: [t.$Identifier, t.$Colon], resolver: ({value}) => `${ansi(...identifierColor)(value)}:` },
+  { left: t.$Line, right: [t.$Bits, t.$Number], resolver: (_, n) => `${ansi(...keywordColor).bold('bits')} ${ansi(...numberColor)(n.value)}`},
+  { left: t.$Line, right: [t.$Default, t.$Rel], resolver: () => `${ansi(...keywordColor).bold('default')} ${ansi(...keywordColor).bold('rel')}`},
 
   // actual instructions
+  { left: t.$Line, right: [t.$Push, t.$DataSize, t.$LBracket, t.$Rel, t.$Identifier, t.$RBracket],
+    resolver: (_, size, __, ___, identifier) => `  ${ansi(...instructionColor)('push')} ${size} ${ansi(...pointerColor)('[')}${ansi(...keywordColor).bold('rel')} ${ansi(...identifierColor)(identifier.value)}${ansi(...pointerColor)(']')}` },
   { left: t.$Line, right: [t.$Push, t.$Value], resolver: (_, v) => `  ${ansi(...instructionColor)('push')} ${v}` },
   { left: t.$Line, right: [t.$Pop, t.$Value], resolver: (_, v) => `  ${ansi(...instructionColor)('pop')} ${v}` },
   { left: t.$Line, right: [t.$Cmp, t.$Register, t.$Comma, t.$Value],
@@ -52,5 +56,10 @@ export default new Grammar([
   { left: t.$Value, right: [t.$Identifier], resolver: (v) => ansi(...identifierColor)(v.value) },
 
   { left: t.$CompoundString, right: [t.$Number], resolver: (n) => ansi(...numberColor)(n.value) },
-  { left: t.$CompoundString, right: [t.$Number, t.$Comma, t.$CompoundString], resolver: (n, _, ns) => ansi(...numberColor)(n.value) + ',' + ns }
+  { left: t.$CompoundString, right: [t.$Number, t.$Comma, t.$CompoundString], resolver: (n, _, ns) => ansi(...numberColor)(n.value) + ',' + ns },
+
+  { left: t.$DataSize, right: [t.$Word], resolver: (v) => ansi(...keywordColor).bold(v.value) },
+  { left: t.$DataSize, right: [t.$DWord], resolver: (v) => ansi(...keywordColor).bold(v.value) },
+  { left: t.$DataSize, right: [t.$QWord], resolver: (v) => ansi(...keywordColor).bold(v.value) },
+  { left: t.$DataSize, right: [t.$OWord], resolver: (v) => ansi(...keywordColor).bold(v.value) },
 ], t.$Program);

src/util/asm/tokenizer.ts

@@ -4,28 +4,58 @@ import {
   $Newline,
 } from "./../../earley";
 
-export default createTokenizer([
-  { match: /^[\r\t ]{1,}$/, token: null },
-  { match: 'section', token: tokens.$Section },
-  { match: 'db', token: tokens.$Db },
-  { match: 'global', token: tokens.$Global },
-  { match: '\n', token: $Newline },
-  { match: ':', token: tokens.$Colon },
-  { match: ',', token: tokens.$Comma },
-  { match: '[', token: tokens.$LBracket },
-  { match: ']', token: tokens.$RBracket },
-  { match: '-', token: tokens.$Minus },
-  { match: 'mov', token: tokens.$Mov },
-  { match: 'push', token: tokens.$Push },
-  { match: 'pop', token: tokens.$Pop },
-  { match: 'call', token: tokens.$Call },
-  { match: 'syscall', token: tokens.$Syscall },
-  { match: 'ret', token: tokens.$Ret },
-  { match: 'je', token: tokens.$Je },
-  { match: 'jmp', token: tokens.$Jmp },
-  { match: 'cmp', token: tokens.$Cmp },
-  { match: 'inc', token: tokens.$Inc },
-  { match: /^[0-9]{1,}$/, token: tokens.$Number },
-  { match: /^(rbp|rsp|rax|rcx|rbx|rdx|rdi|rsi|al|bl|cl|dl|ah|bh|ch|dh|ax|bx|cx|dx|eax|ebx|ecx|edx)$/, token: tokens.$Register },
-  { match: /^[A-Za-z._][A-Za-z_]{0,}$/, token: tokens.$Identifier },
+const asmTokenizer = createTokenizer([
+  // whitespaces
+  [ /^[\r\t ]{1,}/, null],
+  [ /^\n/, $Newline],
+
+  // keywords
+  [ /^section/, tokens.$Section],
+  [ /^db/, tokens.$Db],
+  [ /^global/, tokens.$Global],
+  [ /^bits/, tokens.$Bits],
+  [ /^default/, tokens.$Default],
+  [ /^rel/, tokens.$Rel],
+  [ /^word/, tokens.$Word],
+  [ /^dword/, tokens.$DWord],
+  [ /^qword/, tokens.$QWord],
+  [ /^oword/, tokens.$OWord],
+  
+  // punctuation
+  [ /^:/, tokens.$Colon],
+  [ /^,/, tokens.$Comma],
+  [ /^\[/, tokens.$LBracket],
+  [ /^\]/, tokens.$RBracket],
+  [ /^-/, tokens.$Minus],
+
+  // instructions
+  [ /^mov/, tokens.$Mov],
+  [ /^push/, tokens.$Push],
+  [ /^pop/, tokens.$Pop],
+  [ /^syscall/, tokens.$Syscall],
+  [ /^ret/, tokens.$Ret],
+  [ /^je/, tokens.$Je],
+  [ /^jmp/, tokens.$Jmp],
+  [ /^cmp/, tokens.$Cmp],
+  [ /^inc/, tokens.$Inc],
+
+  // pseudo-instructions
+  [ /^call/, tokens.$Call],
+
+  // 8 bit general purpose registers...
+  [ /^(al|ah|bl|bh|cl|ch|dl|dh)/, tokens.$Register ],
+  // 16 bit general purpose registers...
+  [ /^(ax|bx|cx|dx)/, tokens.$Register ],
+  // 32 bit general purpose registers...
+  [ /^(eax|ebx|ecx|edx)/, tokens.$Register ],
+  // 64 bit general purpose registers...
+  [ /^(rax|rbx|rcx|rdx)/, tokens.$Register ],
+  // other registers, idk.
+  [ /^(rbp|rsp|rdi|rsi)/, tokens.$Register],
+
+  // user-defined
+  [ /^[0-9]{1,}/, tokens.$Number],
+  [ /^0x[0-9A-Fa-f]{1,}/, tokens.$Number],
+  [ /^[A-Za-z._][A-Za-z_]{0,}/, tokens.$Identifier]
 ])
+export default asmTokenizer;

src/util/asm/tokens.ts

@@ -21,6 +21,13 @@ export class $RBracket extends Terminal { }
 export class $Comma extends Terminal { }
 export class $Colon extends Terminal { }
 export class $Minus extends Terminal { }
+export class $Bits extends Terminal { }
+export class $Default extends Terminal { }
+export class $Rel extends Terminal { }
+export class $Word extends Terminal { }
+export class $DWord extends Terminal { }
+export class $QWord extends Terminal { }
+export class $OWord extends Terminal { }
 
 // varying tokens
 export class $Identifier extends Terminal { }
@@ -34,3 +41,4 @@ export class $PointerDereference extends NonTerminal { }
 export class $Program extends NonTerminal { }
 export class $CompoundString extends NonTerminal { }
 export class $Value extends NonTerminal { }
+export class $DataSize extends NonTerminal { }

src/util/asmLogger.ts

@@ -1,3 +0,0 @@
-export function logASM(asm: string) {
-
-}

src/util/disco/tokenizer.ts

@@ -0,0 +1,9 @@
+import { $Newline } from "../../earley";
+import { createTokenizer } from "../generalTokenizer";
+import * as t from './tokens';
+
+export default createTokenizer([
+  [ /^[\r\t ]{1,}/, null],
+  [ /^\n/, $Newline],
+  [/[a-zA-Z][A-Za-z0-9]{0,}/, t.$Identifier],
+])

src/util/disco/tokens.ts

@@ -0,0 +1,18 @@
+import { NonTerminal, Terminal } from "../../earley";
+
+export class $KeywordLink extends Terminal { }
+export class $KeywordEquals extends Terminal { }
+export class $KeywordLParen extends Terminal { }
+export class $KeywordRParen extends Terminal { }
+export class $KeywordConst extends Terminal { }
+
+export class $String extends Terminal {}
+export class $Identifier extends Terminal {}
+
+export class $Program extends NonTerminal { }
+export class $Statement extends NonTerminal { }
+export class $LinkStatement extends NonTerminal { }
+export class $VariableDeclaration extends NonTerminal { }
+export class $Expression extends NonTerminal { }
+export class $InvocationExpression extends NonTerminal { }
+export class $VariableReference extends NonTerminal { }

src/util/generalTokenizer.ts

@@ -1,72 +1,83 @@
-import { TerminalTokenClass } from "../earley";
-import { inspect } from 'util';
+import { Terminal, TerminalTokenClass } from "../earley";
+import { Matcher } from "./regex";
 
-interface TokenMatcher {
-  match: RegExp | string,
-  token: TerminalTokenClass
+type TokenMatcher = [ RegExp, TerminalTokenClass ];
+type Index = number;
+
+interface Match {
+  regex: RegExp;
+  length: number;
+  tokenClass: TerminalTokenClass;
+  matchedString: string;
+}
+
+// this is kinda bullshit lol exec is a dumb method.
+function getFirstMatch(r: RegExp | Matcher, str: string): [Index, string] {
+  if (r instanceof RegExp) {
+    let matches = str.match(r);
+    if(matches === null) return [-1, ''];
+    return [matches.index, matches[0]];
+  }
+}
+
+const getMatchesFromTokenMatcher =
+  (str: string) =>
+  ([regex, tokenClass]: TokenMatcher): Match =>
+{
+  const [index, match] = getFirstMatch(regex, str);
+  if(index === -1) return null;
+  return {
+    regex,
+    tokenClass,
+    length: match.length,
+    matchedString: match
+  }
+}
+
+const advanceLC = (l: number, c: number, str: string) => {
+  for(const char of str) {
+    c ++;
+    if(char === '\n') {
+      l ++;
+      c = 1;
+    }
+  }
+  return [l, c];
 }
 
 export function createTokenizer(tokenMap: TokenMatcher[]) {
-  return function tokenize(str: string) {
-    let tokens = [];
-    let token = '';
-    let line = 1, column = 0;
-    for(let i = 0; i < str.length; i ++) {
-      const char = str[i];
-      const lookahead = (i < str.length - 1 ? str[i + 1] : null)
-      column++;
-      token += char;
 
-      for(const {match: matcher, token: tokenClass} of tokenMap) {
-        if(typeof matcher === 'string') {
-          if(matcher === token) {
-            if(tokenClass !== null) {
-              tokens.push(new tokenClass(line, column - token.length + 1, token));
-            }
-            token = '';
-          } else {
-            // dw about it
-          }
-        } else {
-          // matcher is regex...
-          // * note: this only tests if token contains a match, not that it _is_ a match
-          if(matcher.test(token)) {
-            if(lookahead) {
-              if(!matcher.test(token + lookahead)) {
-                // the next character would not match, so this must be the match.
-                // ! PS: it is possible that even though this would no longer
-                // ! match, another matcher could still match more.
-                // ! in those cases, we would want to expand on this logic
-                // ! to only match if there are no matches for any matcher
-                // ! in the lookahead.
-                // ! in practice this means tracking all possible non lookahead
-                // ! matches, then testing them for their lookahead afterwards
-                // ! in another loop, and only tokenizing if you have only one
-                // ! option, and that option will fail on the lookahead.
-                if(tokenClass !== null) {
-                  tokens.push(new tokenClass(line, column - token.length + 1, token));
-                }
-                token = '';
-              } else {
-                // the lookahead matches this too, so we should probably hold off
-                // on tokenizing it...
-              }
-            } else {
-              if(tokenClass !== null) {
-                tokens.push(new tokenClass(line, column - token.length + 1, token));
-              }
-              token = '';
-            }
-          }
-        }
-      }
+  return function tokenize(str: string, l = 1, c = 1): Terminal[] {
 
-      if(char === '\n') {
-        line ++;
-        column = 0;
-      }
-    }
+    const possibleMatches: Match[] = tokenMap
+      .map(getMatchesFromTokenMatcher(str))
+      .filter(v => !!v);
 
-    return tokens;
+    const longestLength = possibleMatches
+      .map(v => v.length)
+      .reduce((a, v) => a > v ? a : v, -Infinity);
+
+    const longestMatches = possibleMatches
+      .filter(v => v.length === longestLength);
+
+    console.assert(longestMatches.length > 0, 'No token matches found');
+    if(longestMatches.length === 0) process.exit(1);
+
+    const {tokenClass, matchedString} = longestMatches[0];
+    const length = matchedString.length;
+    const rest = str.substring(length);
+
+    
+    const token = tokenClass ? new tokenClass(l, c, matchedString) : null;
+    if(rest === '') return [ token ];
+
+    [l, c] = advanceLC(l, c, str);
+    if(tokenClass) {
+      return [
+        new tokenClass(l, c, matchedString),
+        ...tokenize(rest, l, c)
+      ]
+    }
+    return token ? [token, ...tokenize(rest, l, c)] : tokenize(rest, l, c);
   }
 }

src/util/regex.ts

@@ -0,0 +1,176 @@
+type Match = {
+  offset: number;
+  length: number;
+  text: string;
+  original: string;
+}
+
+const match = (offset: number, length: number, text: string, original: string): Match => {
+  return { offset, length, text, original };
+}
+
+export type Matcher = (str: string) => Match[]
+
+export const matchChar = (char: string): Matcher => {
+  const matcher = (test: string) => {
+    return test[0] === char[0] ? [match(0, 1, test[0], test)] : []
+  }
+  matcher.toString = () => {
+    return char;
+  }
+  return matcher;
+}
+
+export const matchCharClass = (chars: string[]): Matcher => {
+  const matcher = (test: string) => {
+    return chars.includes(test[0]) ? [match(0, 1, test[0], test)] : []
+  }
+  matcher.toString = () => {
+    return '[' + chars.join('') + ']';
+  }
+  return matcher;
+}
+
+const combineMatches = (a: Match, b: Match): Match => {
+  return match(
+    Math.min(a.offset, b.offset),
+    a.length + b.length,
+    a.text + b.text,
+    a.original.length > b.original.length ? a.original : b.original
+  )
+}
+
+export const matchSequence = (matcherA: Matcher, matcherB: Matcher): Matcher => {
+  const matcher = (test: string) => {
+    const matches = [];
+    for (const match of matcherA(test)) {
+      const rest = test.substring(match.length);
+      for (const restMatch of matcherB(rest)) {
+        matches.push(combineMatches(match, restMatch));
+      }
+    }
+    return matches;
+  }
+  matcher.toString = () => {
+    return matcherA.toString() + matcherB.toString();
+  }
+  return matcher;
+}
+
+const repeatMatcher = (matcher: Matcher, test: string, n: number): Match[] => {
+  if(n === 0) {
+    return [match(0, 0, '', test)];
+  }
+  const matches = matcher(test);
+  if(n === 1) {
+    return matches;
+  }
+  return matches.map(match => {
+    const rest = match.original.substring(match.length);
+    return repeatMatcher(matcher, rest, n - 1).map(nextMatch => combineMatches(match, nextMatch));
+  }).flat();
+}
+
+// this logic sucks lol
+// really you should just keep matching until you
+// have no more characters or you hit the match limit.
+// like this shit increases O by 2 on each nested call...
+// TODO /\ \/ /\ \/ /\ \/ /\ \/ /\ \/ /\ \/ /\ \/ /\
+export const matchMany = (matcherA: Matcher, min = 1, max = Infinity): Matcher => {
+  const matcher = (test: string) => {
+    const rmatches: Match[] = [];
+    const limitedMax = Math.min(max, test.length);
+    for(let c = min; c <= limitedMax; c ++) {
+      const matches = repeatMatcher(matcherA, test, c);
+      rmatches.push(...matches);
+    }
+    return rmatches;
+  }
+  matcher.toString = () => {
+    return '(' + (matcherA.toString()) + '){' + (min === 0 ? '' : min) + ',' + (max === Infinity ? '' : max) + '}';
+  }
+  return matcher;
+}
+
+// variable names regex, theory...
+
+
+const matchers = [
+  matchChar('a'),
+  matchCharClass(['a', 'b', 'c']),
+  matchSequence(
+    matchChar('a'),
+    matchCharClass(['a', 'b', 'c'])
+  ),
+  matchMany(
+    matchCharClass(['a', 'b', 'c'])
+  ),
+  matchMany(
+    matchCharClass(['a', 'b', 'c']),
+    1,
+    1
+  ),
+];
+
+const tests = [
+  'a',
+  'b',
+  'c',
+  'd',
+  'ab',
+  'bc',
+  'cd',
+  'da',
+]
+
+console.clear();
+
+const logMatches = (ms: Match[]) => {
+  for(const match of ms) {
+    console.log(
+      ' '.repeat(8) +
+      chalk.white(match.original.substring(0, match.offset)) +
+      chalk.green(match.text) + 
+      chalk.white(match.original.substring(match.offset + match.length))
+    );
+  }
+}
+
+const Y = true;
+const N = false;
+const testMatrix = [
+  [Y, N, N, N, N, N, N, N],
+  [Y, Y, Y, N, N, N, N, N],
+  [N, N, N, N, Y, N, N, N],
+  [Y, Y, Y, N, Y, Y, N, N],
+  [Y, Y, Y, N, N, N, N, N]
+]
+import * as chalk from 'chalk';
+// dirty levels off the CHARTS
+let i = 0, j = 0, p = 0, f = 0;
+for (const matcher of matchers) {
+  j = 0;
+  for (const testString of tests) {
+    const matches = matcher(testString).filter(match => match.length === testString.length);
+    if (matches.length > 0 === testMatrix[i][j]) {
+      p ++;
+    } else {
+      f ++;
+      console.log(
+        chalk.red('[ FAIL ]'),
+        chalk.ansi256(143)('/' + matcher.toString() + '/'),
+        'incorrectly returned',
+        matches.length,
+        'match' + (matches.length !== 1 ? 'es' : '') + ' for',
+        testString,
+      );
+      logMatches(matches);
+      console.log('')
+    }
+    j++;
+  }
+  i++
+}
+console.log('' + p + ' test' + (p !== 1 ? 's' : '') + ' passed.')
+console.log('' + f + ' test' + (f !== 1 ? 's' : '') + ' failed.')
+process.exit(f);

src/util/tokenTest.ts

@@ -6,10 +6,10 @@ class $Plus extends Terminal { }
 class $Newline extends Terminal { }
 
 const tokenizer = createTokenizer([
-  { match: /^[0-9]{1,}$/,       token: $Number  },
-  { match: /^[\r\t ]{1,}$/,     token: null     },
-  { match: '\n',                token: $Newline },
-  { match: '+',                 token: $Plus    },
+  [ /^[0-9]{1,}$/,       $Number  ],
+  [ /^[\r\t ]{1,}$/,     null     ],
+  [ /\n/,                $Newline ],
+  [ /+/,                 $Plus    ],
 ])
 
 console.log(tokenizer("5 + \n 6   ").map(v => v.toString()).join('  '));

todo.md

@@ -1,13 +1,42 @@
 # Todo List
 
-[x] colorize the assembly output
-[ ] rewrite disco tokenizer to the new generalTokenizer
-[ ] add number support
-[ ] add comment support
-[ ] add fixed length array support
-[ ] organize AST elements into classes
-[ ] better logging of the AST
-[ ] optionally artifically slow down compilation (for fun)
-[ ] implement some basic maths operations
-[ ] implement multi-argument invocations
-[ ] implement return values
+- [x] colorize the assembly output
+- [x] create generalTokenizer to make tokenization generic
+- [ ] rewrite disco tokenizer to the new generalTokenizer
+- [ ] explore defining non terminals in a grammar with just a string
+  - possibly using tagged template strings?? 
+- [ ] add an EOF token to earley, and yknow, add it to the tokenizer.
+- [ ] add number support in consts
+- [ ] add comment support
+- [ ] add fixed length array support
+- [ ] organize AST elements into classes
+- [ ] better logging of the AST
+- [ ] optionally artifically slow down compilation (for fun)
+- [ ] implement functions
+- [ ] implement some basic maths operations
+- [ ] implement multi-argument invocations
+- [ ] implement return values
+- [ ] write a regex compiler
+- [ ] write log in disco. creat a library for just doing syscalls. the rest can be done in disco
+
+# Changelog
+
+- fixed macos compilation to use relative addressing (i think)
+- fixed a bug in the general tokenizer that failed to match some tokens properly.
+
+---
+
+- create generalized tokenizer
+- implement assembly language grammar for syntax highlighting
+- create a vscode extension for syntax highlighting
+
+---
+
+- compile disco code to assembly as POC
+- create an AST for disco code
+- implement earley grammar for disco including:
+  - linking library functions
+  - calling functions
+  - string literals
+  - string variables
+- created earley parser