HMR and semi smart bandits

2021-12-22 17:59:56 -05:00 · 2021-12-22 17:59:56 -05:00 · 5f6e26cdb0
parent 52b5f5ef92
commit 5f6e26cdb0
9 changed files with 262 additions and 34 deletions
--- a/.system
+++ b/.system
@ -4,7 +4,7 @@
    "B7FFCD6C11974FC7A74509B6683D7420": {
      "config": {
        "sshd": "invoke sshd start",
-        "invoke": "invoke EXPLORE run 10"
+        "bandit-agent": "invoke explore run 10"
      },
      "module": "systemd"
    },
@ -50,11 +50,11 @@
    "7E884FCACAC74FDF985D46B2BC937B5B": {
      "config": {
        "bandits": [
-          "SLOT1",
-          "SLOT2",
-          "SLOT3",
-          "SLOT4",
-          "SLOT5"
+          "slot1",
+          "slot2",
+          "slot3",
+          "slot4",
+          "slot5"
        ]
      },
      "module": "n-arm-bandit/agent/exploreOnly"
@ -63,12 +63,12 @@
  "aliases": {
    "systemd": "B7FFCD6C11974FC7A74509B6683D7420",
    "sshd": "600F5B3828BC4D78BCBA375596F6898B",
-    "SLOT1": "51E27DEEE43B4731A700D1E01E4CD27A",
-    "SLOT2": "05EA8268E28F4C9FAC7F8E5BE0ABAB5F",
-    "SLOT3": "0F0B6AEB69754319A7E480E3989F7E54",
-    "SLOT4": "34064741B6324D9BBCEE0A1364F21220",
-    "SLOT5": "18461538E5A1487EBC9F54040295440A",
-    "EXPLORE": "7E884FCACAC74FDF985D46B2BC937B5B"
+    "slot1": "51E27DEEE43B4731A700D1E01E4CD27A",
+    "slot2": "05EA8268E28F4C9FAC7F8E5BE0ABAB5F",
+    "slot3": "0F0B6AEB69754319A7E480E3989F7E54",
+    "slot4": "34064741B6324D9BBCEE0A1364F21220",
+    "slot5": "18461538E5A1487EBC9F54040295440A",
+    "explore": "7E884FCACAC74FDF985D46B2BC937B5B"
  },
-  "devMode": false
+  "devMode": true
 }
--- a/loader.mjs
+++ b/loader.mjs
@ -4,6 +4,7 @@ import chalk from 'chalk';
 import { dirname, resolve as pathResolve } from 'path';
 import { fileURLToPath } from 'url';
 import { sep } from 'path';
+import * as uuid from 'uuid';

 const __dirname = dirname(fileURLToPath(import.meta.url));

@ -39,7 +40,7 @@ export async function resolve(specifier, context, defaultResolver) {
    '@commands:save':     pathResolve(__dirname, 'src', 'commands', 'save.ts'),
    '@commands:help':     pathResolve(__dirname, 'src', 'commands', 'help.ts'),

-    '@builtin:':   (s) => pathResolve(__dirname, 'src', 'modules', s) + '.ts',
+    '@builtin:':   (s) => pathResolve(__dirname, 'src', 'modules', s) + '.ts?hmr=' + uuid.v4(),

    '@echo off':          pathResolve(__dirname, 'src', 'noop.ts')
  };
--- a/package.json
+++ b/package.json
@ -9,11 +9,13 @@
    "tsc": "tsc"
  },
  "dependencies": {
+    "@types/chokidar": "^2.1.3",
    "@types/lodash": "^4.14.178",
    "@types/md5": "^2.3.1",
    "@types/node": "^16.11.12",
    "@types/uuid": "^8.3.3",
    "chalk": "^5.0.0",
+    "chokidar": "^3.5.2",
    "lodash": "^4.17.21",
    "md5": "^2.3.0",
    "serverline": "^1.5.0",
--- a/src/commands/create.ts
+++ b/src/commands/create.ts
@ -21,20 +21,8 @@ export default async function create(module: string, name: string, id: string) {
      throw new Error('MODULE_ALIAS_TAKEN');
    }
  }
-  const imported = (await import('@builtin:' + module));
-  const { default: moduleOptions, ...functions} = imported;

-  const config: object = (function () {
-    if(moduleOptions && 'config' in moduleOptions) {
-      if(typeof moduleOptions.config === 'function') {
-        return moduleOptions.config();
-      } else {
-        return moduleOptions.config;
-      }
-    } else {
-      return {};
-    }
-  })();
+  const [functions, config] = await loadModule(module);

  id ??= uuid.v4().replace(/-/g, '').toUpperCase();
  system.instances.set(id, {
@ -55,3 +43,24 @@ export default async function create(module: string, name: string, id: string) {
  console.log('      Id:', chalk.ansi256(242)(id));
  return id;
 }
+
+export async function loadModule(module: string): Promise<[functions: any, config: any]> {
+
+  const imported = (await import('@builtin:' + module));
+  const { default: moduleOptions, ...functions} = imported;
+  
+  const config: object = (function () {
+    if(moduleOptions && 'config' in moduleOptions) {
+      if(typeof moduleOptions.config === 'function') {
+        return moduleOptions.config();
+      } else {
+        return moduleOptions.config;
+      }
+    } else {
+      return {};
+    }
+  })();
+  return [
+    functions, config
+  ]
+}
--- a/src/index.ts
+++ b/src/index.ts
@ -3,16 +3,23 @@
 import { Instance, ParsedSystemState, System } from '@kernel:base';
 import '@kernel:log-hook';
 import createExecutor from '@commands:executor';
-import create from '@commands:create';
+import create, { loadModule } from '@commands:create';
 import ls from '@commands:ls';
 import save from '@commands:save';
 import help from '@commands:help';
 import * as uuid from 'uuid';
 import serverline from 'serverline';
 import { existsSync, readFileSync, writeFileSync } from 'fs';
-import { resolve } from 'path'
+import { dirname, resolve } from 'path'
 import chalk from 'chalk';
 import md5 from 'md5';
+import { fileURLToPath } from 'url';
+import dar from 'chokidar';
+import _ from 'lodash';
+const { merge } = _;
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const builtins = resolve(__dirname, 'modules');

 const args = process.argv.slice(2);
 const [ startupFile ] = args;
@ -119,6 +126,7 @@ const executor = createExecutor(kernel);
  if(existsSync('.system')) {
    const state: ParsedSystemState = JSON.parse(readFileSync('.system').toString());
    system.handoff = state.handoff;
+    system.devMode = state.devMode;
    for(const [id, info] of Object.entries(state.instances)) {
      const [alias] = Object.entries(state.aliases).find(([,tryId]) => tryId === id) ?? [undefined];
      await kernel.create(info.module, alias, id);
@ -149,9 +157,37 @@ const executor = createExecutor(kernel);
  serverline.on('SIGINT', () => exec('quit'));
  console.log('For help, type help');

+  dar.watch(builtins, {
+    ignoreInitial: true,
+    cwd: builtins
+  }).on('all', async (type, path, stats) => {
+    const fqdn = path.substring(0, path.length - 3);
+    if(system.devMode) {
+      console.log(chalk.ansi256(202)('/ ') + chalk.ansi256(242)(fqdn));
+      try {
+        const [functions, config] = await loadModule(fqdn);
+        let count = 0;
+        for(const [id, instance] of system.instances.entries()) {
+          if(instance.module === fqdn) {
+            instance.functions = functions;
+            instance.privateScope.config = merge(config, instance.privateScope.config);
+            count ++;
+          }
+        }
+        console.log(chalk.ansi256(34)('/ ') + chalk.ansi256(242)('Reloaded ' + count + ' instance' + (count === 1 ? '' : 's')));;
+      } catch (e) {
+        console.log(e);
+      }
+    } else {
+      console.log('devmode is off');
+    }
+  });
+
 })().catch((e: Error) => {
  console.error(e);
+  process.exit(1);
 });

 checkpoint('Kernel Loaded');
 import '@echo off';
+
--- a/src/modules/n-arm-bandit/agent/exploreOnly.ts
+++ b/src/modules/n-arm-bandit/agent/exploreOnly.ts
@ -6,13 +6,71 @@ export default {
  }
 }

+function weightedAverage(average: number, newDatum: number, previousAverages: number) {
+  return (average * previousAverages + newDatum) / (previousAverages + 1);
+}
+
 export async function run(tries: number) {
+  const data: {
+    pulls: {
+      [name: string]: number
+    },
+    averages: {
+      [name: string]: number
+    },
+    average: number
+  } = {
+    pulls: {},
+    averages: {},
+    average: 0
+  };
+  let rewardTotal = 0;
+
+  for(const slot of this.config.bandits) {
+    data.pulls[slot] = 0;
+    data.averages[slot] = 0;
+  }
+  data.average = 0;
+  
  for(let i = 0; i < tries; i ++) {
-    const slot = this.config.bandits[Math.floor(Math.random() * this.config.bandits.length)];
-    const result = await exec(`invoke ${slot} pull`);
+
+    const slot = (() => {
+      const noDataSlots = this.config.bandits.filter((bandit: string) => data.pulls[bandit] === 0);
+      if(noDataSlots.length > 0) return noDataSlots[Math.floor(Math.random() * noDataSlots.length)]
+      return this.config.bandits.map((s: string) => {
+        return {
+          name: s,
+          rating: data.averages[s]
+        }
+      }).sort((a: any, b: any) => a.rating > b.rating)[0].name;
+    })();
+
    console.log(`Pulling ${slot}...`);
+    const result = await exec(`invoke ${slot} pull`);
+    rewardTotal += result;
+    data.averages[slot] = weightedAverage(data.averages[slot], result, data.pulls[slot]);
+    data.pulls[slot] ++;
    console.log(`Got ${result.toFixed(2)} reward`);
  }
+
+  console.log('Sim finished with total reward ' + rewardTotal);
+
+  const trueAverages: any = {};
+  let maxSlot = null;
+  let maxAvg = -Infinity;
+
+  for(const slot of this.config.bandits) {
+    const avg = await exec(`invoke ${slot} getAverageReward`);
+    if(avg > maxAvg) {
+      maxSlot = slot;
+      maxAvg = avg;
+    }
+  }
+
+  const maxReward = maxAvg * tries;
+  const regret = maxReward - rewardTotal;
+  console.log('Regret percent: ' + ((regret / maxReward) * 100).toFixed(2) + '%');
+
 }

 export function addBandit(name: string) {
--- a/src/modules/n-arm-bandit/bandit.ts
+++ b/src/modules/n-arm-bandit/bandit.ts
@ -19,3 +19,11 @@ function randomNormal(width: number = 1, offset: number = 0) {
 export function pull() {
  return randomNormal(this.config.variance, this.config.base);
 }
+
+export function getAverageReward() {
+  const avg = this.config.base;
+  this.config.variance = Math.random() * 10;
+  this.config.base = Math.random() * 10;
+  console.log('slots real reward was: ' + avg);
+  return avg;
+}
--- a/src/modules/systemd.ts
+++ b/src/modules/systemd.ts
@ -8,7 +8,12 @@ import { exec } from '@kernel:base';

 export async function boot() {
  for(const [name, script] of Object.entries(this.config)) {
-    await exec(script as string);
+    try {
+      await exec(script as string);
+    } catch (e) {
+      console.log('systemd startup script \'' + name + '\' failed with error:')
+      console.log(e);
+    }
  }
 }

--- a/yarn.lock
+++ b/yarn.lock
@ -34,6 +34,13 @@
  resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e"
  integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==

+"@types/chokidar@^2.1.3":
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/@types/chokidar/-/chokidar-2.1.3.tgz#123ab795dba6d89be04bf076e6aecaf8620db674"
+  integrity sha512-6qK3xoLLAhQVTucQGHTySwOVA1crHRXnJeLwqK6KIFkkKa2aoMFXh+WEi8PotxDtvN6MQJLyYN9ag9P6NLV81w==
+  dependencies:
+    chokidar "*"
+
 "@types/json5@^0.0.29":
  version "0.0.29"
  resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
@ -71,6 +78,14 @@ acorn@^8.4.1:
  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.6.0.tgz#e3692ba0eb1a0c83eaa4f37f5fa7368dd7142895"
  integrity sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw==

+anymatch@~3.1.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716"
+  integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==
+  dependencies:
+    normalize-path "^3.0.0"
+    picomatch "^2.0.4"
+
 arg@^4.1.0:
  version "4.1.3"
  resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
@ -90,6 +105,18 @@ bcrypt-pbkdf@^1.0.2:
  dependencies:
    tweetnacl "^0.14.3"

+binary-extensions@^2.0.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
+  integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
+
+braces@~3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
+  integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
+  dependencies:
+    fill-range "^7.0.1"
+
 chalk@^5.0.0:
  version "5.0.0"
  resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.0.0.tgz#bd96c6bb8e02b96e08c0c3ee2a9d90e050c7b832"
@ -100,6 +127,21 @@ charenc@0.0.2:
  resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"
  integrity sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=

+chokidar@*, chokidar@^3.5.2:
+  version "3.5.2"
+  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.2.tgz#dba3976fcadb016f66fd365021d91600d01c1e75"
+  integrity sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ==
+  dependencies:
+    anymatch "~3.1.2"
+    braces "~3.0.2"
+    glob-parent "~5.1.2"
+    is-binary-path "~2.1.0"
+    is-glob "~4.0.1"
+    normalize-path "~3.0.0"
+    readdirp "~3.6.0"
+  optionalDependencies:
+    fsevents "~2.3.2"
+
 cpu-features@0.0.2:
  version "0.0.2"
  resolved "https://registry.yarnpkg.com/cpu-features/-/cpu-features-0.0.2.tgz#9f636156f1155fd04bdbaa028bb3c2fbef3cea7a"
@ -122,11 +164,54 @@ diff@^4.0.1:
  resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
  integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==

+fill-range@^7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
+  integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==
+  dependencies:
+    to-regex-range "^5.0.1"
+
+fsevents@~2.3.2:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
+glob-parent@~5.1.2:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
+  integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
+  dependencies:
+    is-glob "^4.0.1"
+
+is-binary-path@~2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09"
+  integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==
+  dependencies:
+    binary-extensions "^2.0.0"
+
 is-buffer@~1.1.6:
  version "1.1.6"
  resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
  integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==

+is-extglob@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
+  integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=
+
+is-glob@^4.0.1, is-glob@~4.0.1:
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
+  integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==
+  dependencies:
+    is-extglob "^2.1.1"
+
+is-number@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b"
+  integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==
+
 json5@^1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe"
@ -163,6 +248,23 @@ nan@^2.14.1, nan@^2.15.0:
  resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee"
  integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==

+normalize-path@^3.0.0, normalize-path@~3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
+  integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
+
+picomatch@^2.0.4, picomatch@^2.2.1:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972"
+  integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
+
+readdirp@~3.6.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7"
+  integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==
+  dependencies:
+    picomatch "^2.2.1"
+
 safer-buffer@~2.1.0:
  version "2.1.2"
  resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@ -193,6 +295,13 @@ strip-bom@^3.0.0:
  version "0.0.1"
  resolved "git+https://github.com/TooTallNate/node-telnet.git#780340617e1f223de384cdfcb7cecf0a1a6f1159"

+to-regex-range@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4"
+  integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==
+  dependencies:
+    is-number "^7.0.0"
+
 ts-node@^10.4.0:
  version "10.4.0"
  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.4.0.tgz#680f88945885f4e6cf450e7f0d6223dd404895f7"