From 2baebc7845eae2c1440ee14c6db28484c7beac25 Mon Sep 17 00:00:00 2001 From: Marcus Date: Sat, 24 Apr 2021 00:42:01 -0400 Subject: [PATCH] some stuff --- .github/workflows/publish.yml | 6 +- package.json | 6 +- relay/service.mjs | 440 ++++++++++-------- .../.local/share/valnet/relay/config.json | 1 + .../.local/share/valnet/relay/default.json | 1 + .../valnet/relay/shy-fly-72-endpoints.json | 1 + yarn.lock | 12 + 7 files changed, 269 insertions(+), 198 deletions(-) create mode 100644 undefined/.local/share/valnet/relay/config.json create mode 100644 undefined/.local/share/valnet/relay/default.json create mode 100644 undefined/.local/share/valnet/relay/shy-fly-72-endpoints.json diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 03f6a78..f7e6a95 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,7 +3,7 @@ name: Publish on: push: branches: - - master + - stable jobs: publish: @@ -20,7 +20,7 @@ jobs: - name: Install Node, NPM and Yarn uses: actions/setup-node@v1 with: - node-version: 15 + node-version: 16 - name: Get yarn cache directory path id: yarn-cache-dir-path @@ -52,4 +52,4 @@ jobs: # This is used for uploading release assets to github GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - yarn postinstall && yarn build && yarn electron-builder --publish always --win --mac --linux + yarn prep && yarn build && yarn electron-builder --publish always --win --mac --linux diff --git a/package.json b/package.json index d50d301..50f0db6 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "productName": "ElectronReact", "description": "Electron application boilerplate based on React, React Router, Webpack, React Fast Refresh for rapid application development", "scripts": { - "postinstall": "node -r @babel/register .erb/scripts/CheckNativeDep.js && electron-builder install-app-deps && yarn cross-env NODE_ENV=development webpack --config ./.erb/configs/webpack.config.renderer.dev.dll.babel.js && opencollective-postinstall && yarn-deduplicate yarn.lock", + "prep": "node -r @babel/register .erb/scripts/CheckNativeDep.js && electron-builder install-app-deps && yarn cross-env NODE_ENV=development webpack --config ./.erb/configs/webpack.config.renderer.dev.dll.babel.js && opencollective-postinstall && yarn-deduplicate yarn.lock", "build": "concurrently \"yarn build:main\" \"yarn build:renderer\"", "build:main": "cross-env NODE_ENV=production webpack --config ./.erb/configs/webpack.config.main.prod.babel.js", "build:renderer": "cross-env NODE_ENV=production webpack --config ./.erb/configs/webpack.config.renderer.prod.babel.js", @@ -26,6 +26,7 @@ "start:main": "cross-env NODE_ENV=development electron -r ./.erb/scripts/BabelRegister ./src/main.dev.ts", "start:renderer": "cross-env NODE_ENV=development webpack serve --config ./.erb/configs/webpack.config.renderer.dev.babel.js", "test": "jest", + "supervisor": "supervisor", "relay": "supervisor -w relay,src -n exit relay/index.mjs", "relay:service": "supervisor -w relay,src -- relay/service.mjs", "sloc": "find lib -type f | xargs wc -l" @@ -251,6 +252,7 @@ "electron-debug": "^3.1.0", "electron-log": "^4.2.4", "electron-updater": "^4.3.4", + "express-ws": "^4.0.0", "font-ascii": "^1.2.1", "gradient-string": "^1.2.0", "history": "^5.0.0", @@ -275,7 +277,7 @@ "volatile": "^7.0.1" }, "devEngines": { - "node": ">=10.x", + "node": ">=16.x", "npm": ">=6.x", "yarn": ">=1.21.3" }, diff --git a/relay/service.mjs b/relay/service.mjs index 1089b8b..8d4e19d 100644 --- a/relay/service.mjs +++ b/relay/service.mjs @@ -1,230 +1,284 @@ -import Signale from 'signale'; +// imports that arent installed... import { execSync, spawn } from 'child_process'; -import Datastore from 'nedb'; import { config } from '../src/lib/config/index.js'; -import express from 'express'; -import Volatile from 'volatile'; +import http from 'http'; +import os from 'os'; +import EventEmitter from 'events'; -const logLock = new Volatile({}); - -(async () => { - -const log = Signale.scope('SRVC'); -const branch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); -let proc; -const logs = new Datastore({ - filename: 'svc.log', - autoload: true -}); -const app = express(); - - -try { - logp('Attempting yarn install...') - appendLogs('yarn', execSync(`yarn`)); -} catch (e) { - logp('failed to yarn install...') +let external = {}; +async function doExternalImports() { + external.Signale = (await import('signale')).default; + external.Datastore = (await import('nedb')).default; + external.express = (await import('express')).default; + external.Volatile = (await import('volatile')).default; + external.expressWs = (await import('express-ws')).default; } -logp('=================================='); -logp('Starting Valnet Node as a Service!'); -logp('Syncing to branch: ' + branch); -logp('=================================='); - -setInterval(function update() { - const remoteHash = execSync('git ls-remote https://github.com/marcus13345/valnet.git').toString() - .split('\n') - .filter(test => { - return test.trim().endsWith(branch); - })[0] - .split('\t')[0] - .trim(); - const localHash = execSync(`git rev-parse ${branch}`).toString().trim(); - if(remoteHash !== localHash) { - logp(`remote hash: ${remoteHash}`); - logp(`local hash: ${localHash}`); - - logp('killing relay...'); - try { - proc.kill(); - } catch (e) { - logp('failed to kill active relay...', 'error'); - logp(e, 'error'); - } +(async function bootloader() { + let yarnOutput = ""; + try { + // sanity install packages... + // happens if a package is installed + // and used in this script, without + // restarting the service entirely + // between updates from github. + // this is also why we dynamically + // load the dependencies... + yarnOutput += execSync(`yarn`); + } catch { + enterRecoveryMode(yarnOutput); + return; } -}, 5000); - -(function keepAlive() { - proc = spawn('node', ['./relay/index.mjs'], { - stdio: 'pipe' - }); - appendLogs('relay', 'STARTED', 'event'); - - proc.stdout.on('data', (data) => { - process.stdout.write(data); - appendLogs('relay', data.toString(), 'stdout'); - }); - - proc.stderr.on('data', (data) => { - process.stderr.write(data); - appendLogs('relay', data.toString(), 'stderr'); - }); - - proc.on('exit', () => { - appendLogs('relay', 'STOPPED', 'event'); - logp('relay exitted'); - logp('attempting to fetch new version'); - - appendLogs('fetch', execSync(`git fetch`)); - appendLogs('update', execSync(`git pull`)); - appendLogs('yarn', execSync(`yarn`)); - - logp('restarting...') - setTimeout(() => { - keepAlive(); - }, 1000); - }) + try { + await doExternalImports(); + } catch (e) { + enterRecoveryMode(e.toString()); + return; + } + startService(); })(); -function logp(message, type = 'info') { - log[type](message); - appendLogs('service', message + '\n') -} -function appendLogs(source, data, type = 'output') { - logLock.lock(function(lock) { - return new Promise(res => { - logs.insert({ +async function startService() { + const logLock = new external.Volatile({}); + const log = external.Signale.scope('SRVC'); + const branch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); + let proc; + const logs = new external.Datastore({ + filename: 'svc.log', + autoload: true + }); + const app = external.express(); + external.expressWs(app); + const logEvents = new EventEmitter(); + + logp('=================================='); + logp('Starting Valnet Node as a Service!'); + logp('Syncing to branch: ' + branch); + logp('==================================='); + logp('= = = = = = = = = = = = = = = = = ='); + logp('= = = = = = = = = = = = '); + logp('= = = = = = = = = '); + logp('= = = = = = = '); + logp('= = = = = = '); + + setInterval(function update() { + const remoteHash = execSync('git ls-remote https://github.com/marcus13345/valnet.git').toString() + .split('\n') + .filter(test => { + return test.trim().endsWith(branch); + })[0] + .split('\t')[0] + .trim(); + const localHash = execSync(`git rev-parse ${branch}`).toString().trim(); + if(remoteHash !== localHash) { + logp(`remote hash: ${remoteHash}`); + logp(`local hash: ${localHash}`); + + logp('killing relay...'); + try { + proc.kill(); + } catch (e) { + logp('failed to kill active relay...', 'error'); + logp(e, 'error'); + } + } + }, 5000); + + (function keepAlive() { + proc = spawn('node', ['./relay/index.mjs'], { + stdio: 'pipe', + env: { + FORCE_COLOR: true + } + }); + appendLogs('relay', 'STARTED', 'event'); + + proc.stdout.on('data', (data) => { + process.stdout.write(data); + appendLogs('relay', data.toString(), 'stdout'); + }); + + proc.stderr.on('data', (data) => { + process.stderr.write(data); + appendLogs('relay', data.toString(), 'stderr'); + }); + + proc.on('exit', () => { + appendLogs('relay', 'STOPPED', 'event'); + logp('relay exitted'); + logp('attempting to fetch new version'); + + appendLogs('fetch', execSync(`git fetch`)); + appendLogs('update', execSync(`git pull`)); + appendLogs('yarn', execSync(`yarn`)); + + logp('restarting...') + setTimeout(() => { + keepAlive(); + }, 1000); + }) + })(); + + function logp(message, type = 'info') { + log[type](message); + appendLogs('service', message + '\n') + } + + function appendLogs(source, data, type = 'output') { + logLock.lock(function(lock) { + const newDoc = { message: data.toString(), type: type, src: source, timestamp: new Date().getTime() - }, (err, doc) => { - res(lock); + }; + logEvents.emit('all', [newDoc]); + return new Promise(res => { + logs.insert(newDoc, (err, doc) => { + res(lock); + }) }) }) - }) -} + } -function getSessions() { - return new Promise(res => { - logs.find({ - type: 'event', - message: { $in: ['STARTED', 'STOPPED'] } - }, {}, (err, docs) => { - const sessions = []; - let start = null; - for(const event of docs) { - if(event.message === 'STARTED') { - if (start !== null) { + function getSessions() { + return new Promise(res => { + logs.find({ + type: 'event', + message: { $in: ['STARTED', 'STOPPED'] } + }, {}, (err, docs) => { + const sessions = []; + let start = null; + for(const event of docs) { + if(event.message === 'STARTED') { + if (start !== null) { + sessions.push({ + started: start, + stopped: event.timestamp + }); + } + start = event.timestamp; + } + if(event.message === 'STOPPED' && start !== null) { sessions.push({ started: start, stopped: event.timestamp }); + start = null; } - start = event.timestamp; } - if(event.message === 'STOPPED' && start !== null) { - sessions.push({ - started: start, - stopped: event.timestamp - }); - start = null; - } - } - sessions.sort((a, b) => a.started > b.started); - res(sessions); + sessions.sort((a, b) => a.started > b.started); + res(sessions); + }); }); + } + + app.get('/', (req, res) => { + res.end(` + Logs
+ Sessions
+ Restart + `); + }) + + app.get('/restart', async (req, res) => { + proc.kill(); + res.redirect('/'); + }) + + app.get('/logs', (req, res) => { + // res.redirect(`/logs/${Date.now() - (1000 * 60 * 60 * 24)}`) + res.end(Template.realtimeLogs()); + }) + + app.get('/logs/:start/:end', (req, res) => { + + logs.find({ + timestamp: { $gt: parseInt(req.params.time) } + }, {}).sort({ + timestamp: -1 + }).limit(100).exec((err, docs) => { + + res.end(Template.logs(docs.reverse().map(v => v.message))); + + if(err) { + res.end(err.toString()); + return; + } + // ${new Date(logItem.timestamp).toLocaleString().padStart(40)}: + res.end(); + }) }); + + + app.get('/api/sessions', async (req, res) => { + res.json(await getSessions()); + }) + + app.ws('/api/logs', async (ws) => { + logEvents.on('all', broadcast); + function broadcast(docs) { + for(const doc of docs) + ws.send(doc.message); + } + ws.on('close', _ => { + logEvents.off('all', broadcast); + }) + }); + + app.listen(config.ports.service); +}; + +function enterRecoveryMode(message) { + http.createServer((req, res) => { + res.end(Template.recoveryMode(message)); + }).listen(config.ports.service); } -app.get('/', (req, res) => { - res.end(` - Logs
- Sessions
- Restart - `); -}) -app.get('/restart', async (req, res) => { - proc.kill(); - res.redirect('/'); -}) - -app.get('/api/sessions.json', async (req, res) => { - res.json(await getSessions()); -}) - -app.get('/logs', (req, res) => { - res.redirect(`/logs/${Date.now() - (1000 * 60 * 60 * 24)}`) -}) - -app.get('/logs/:time', (req, res) => { - - logs.find({ - timestamp: { $gt: parseInt(req.params.time) } - }, {}).sort({ - timestamp: -1 - }).limit(100).exec((err, docs) => { - - res.end(Template.logs(docs.reverse().map(v => v.message))); - - if(err) { - res.end(err.toString()); - return; - } -// ${new Date(logItem.timestamp).toLocaleString().padStart(40)}: - res.end(); - }) -}); const Template = { logs(messages) { - return ` - - - - - - 
+		return `
+		
+		
 ${messages.join('').replace(/\u001B\[.*?[A-Za-z]/g, '')}
-			

-			






-			
-		
-		`;
+
+





+ `; + }, + realtimeLogs() { + return ` + + 

+		






+		`;
 	},
 	Json(obj) {
 		return `
 ${JSON.stringify(obj, null, 2)}
 		
`
+	},
+	recoveryMode(message) {
+		return `
${os.hostname()} has entered recovery mode...

+		
Last words:

+		
${message}
`
 	}
-};
-
-app.listen(config.ports.service);
-
-
-
-
-})();
\ No newline at end of file
+};
\ No newline at end of file
diff --git a/undefined/.local/share/valnet/relay/config.json b/undefined/.local/share/valnet/relay/config.json
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/undefined/.local/share/valnet/relay/config.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/undefined/.local/share/valnet/relay/default.json b/undefined/.local/share/valnet/relay/default.json
new file mode 100644
index 0000000..a328902
--- /dev/null
+++ b/undefined/.local/share/valnet/relay/default.json
@@ -0,0 +1 @@
+{"cache":[["keyv:name",{"value":"{\"value\":\"shy-fly-72\",\"expires\":null}"}],["keyv:private-key",{"value":"{\"value\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCcN8eGTX54f26/\\nZD+hQz8xzY3DBMh/QEPymh7Hh4BLAMKtlbnvpZ12A7ugPiSsjhQsZhEpkdE+Uo0J\\nrA1GLO0XPxgPP3QLPTv6/L9ZKIzOFEiT30YE7GDSBflj9kyyRNVpJrrgUHxsKmmg\\nQx+HakJDuNynN8LX8KltxzPZO1+Cx0Zy3r6iozniBPustzq03+YceWRhKCwgf2gM\\nXWtUxMY8M++Y/nFWDJ9CQMAhUOduQN/HFSxeWnrGRutQlzuj92Zd2S7p29CNBYry\\nk1D4BYu1cfVHhh0QlgflF7G83qQi9WEpeNwLXyx1ocKqVH4juri5VUI4SCNCuVO2\\nSkAGPqeJAgMBAAECggEAQwzJ7cIbM1r2zN0ZVn0HhbfFTdwitkN+JoWyClAFZ6vW\\n4yxtlweNkxa9FGU31nxQ+lGAskfs+IP0Fx+qvuEPjje9euXwR9arhOmws3LtJJKe\\nKCHf6sMON3bq+NXBS46xzqUZ9qd1D/QQTAIjaTrZYDnbM0aCsa6Q+hZTEb0jHH0B\\nzx0pzalljPo7y146WEAUjzC8PW0aGY80DTrxYYUXjvLEXzHygzLJR9GS///6E2mQ\\nZBmYGgOcbIqb54Fky4hkUSFClHkmpN9QH47+buh5jNnE9WOoWBrDmqzk68mUK7c9\\ns2DV3DUplO1Py1kFYuuGQNf7AK4l2HoRuUpJe6l40QKBgQDRM1tr8hrOtzXxKYOK\\nftvHV2ynUXAVblR6Be4iUo0GlipeZ8o6ijb21Hks9L81vnjB2c2zo80HYZvfYn7Y\\nbLFWpYtuZTO19HEb2Z3qruVJz2bFXRS9fdURSTSVufuBZ0gYFDtGFXvjUpPlv4ww\\nvXEtr4DqoOLIlRcmui806we+BQKBgQC/KivrH2O0G5W5uimGc5t4aVGemWBqJTyw\\ncb5phaYpDh8pJpbhPwW3L13jMX4wqxkW8xOc2EK1vyiLWeJJ9iVO/B5bZnQAAL2G\\nUwDUlKfLGeqEPlkKNW6Om89POZ/xo9jPub565NnG8oUWcCYr9pn8EY3UXkJIlsp7\\nXiiwD6N2tQKBgEhqfLULI5h29yc59ZzVeQKyEpyApknk3T04HEypQsQK6zOuveQx\\nwAEFX8TeG+pgurBv0rLierCZazmSgNIuHd0ehPt78MCFkznOxMleLWS1dJ/RLsLB\\nciX/r0I9FQCgXeZn6nTCLxZrmWKoQcEFcs4buIZ/lsFUSqVPQdQRn6Z5AoGAdUkZ\\ncwNqN6mo/i+CpLQvUfxEEGuH5Pf9uT/AFJvkK1I3uhasAXyaNB8Cmo7WHmQLW7I9\\neUCSBVJIPN8j4D79+uziby93wjyyC0THwKWIIStAYGykUogf0a00zyXKxQXC0wfi\\nHyJjnxVSpEaglN6S8T2P6BkAz+p8Rp6plgOyRfUCgYEAq8miCbC2EDNYbLH8+ViM\\nwsNyHBIFaGXSoY2uWfQTwnt1gWEplGRayJVnc0U4UsUv7jnALZH4aN2yrEK5aOqN\\nMPvVgeOrUTnf+Kltb06vbaCCqhQp4mqmkTFvApkOnnPMf+dAw8w614/ZEEs1wluo\\nzNEFNRrI+iD4kTxeUAszUPI=\\n-----END PRIVATE KEY-----\",\"expires\":null}"}],["keyv:public-key",{"value":"{\"value\":\"-----BEGIN PUBLIC KEY-----\\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnDfHhk1+eH9uv2Q/oUM/\\nMc2NwwTIf0BD8poex4eASwDCrZW576WddgO7oD4krI4ULGYRKZHRPlKNCawNRizt\\nFz8YDz90Cz07+vy/WSiMzhRIk99GBOxg0gX5Y/ZMskTVaSa64FB8bCppoEMfh2pC\\nQ7jcpzfC1/Cpbccz2TtfgsdGct6+oqM54gT7rLc6tN/mHHlkYSgsIH9oDF1rVMTG\\nPDPvmP5xVgyfQkDAIVDnbkDfxxUsXlp6xkbrUJc7o/dmXdku6dvQjQWK8pNQ+AWL\\ntXH1R4YdEJYH5RexvN6kIvVhKXjcC18sdaHCqlR+I7q4uVVCOEgjQrlTtkpABj6n\\niQIDAQAB\\n-----END PUBLIC KEY-----\",\"expires\":null}"}]],"lastExpire":1619217000188}
\ No newline at end of file
diff --git a/undefined/.local/share/valnet/relay/shy-fly-72-endpoints.json b/undefined/.local/share/valnet/relay/shy-fly-72-endpoints.json
new file mode 100644
index 0000000..fd6259e
--- /dev/null
+++ b/undefined/.local/share/valnet/relay/shy-fly-72-endpoints.json
@@ -0,0 +1 @@
+{"cache":[["keyv:cache",{"value":"{\"value\":[\"valnet.xyz:5500\",\"nightly.valnet.xyz:5600\",\"canary.valnet.xyz:5600\",\"stable.valnet.xyz:5600\"],\"expires\":null}"}],["keyv:valnet.xyz:5500",{"value":"{\"value\":{\"host\":\"valnet.xyz\",\"port\":\"5500\",\"lastPing\":null,\"status\":\"unknown\"},\"expires\":null}"}],["keyv:nightly.valnet.xyz:5600",{"value":"{\"value\":{\"host\":\"nightly.valnet.xyz\",\"port\":\"5600\",\"lastPing\":null,\"status\":\"unknown\"},\"expires\":null}"}],["keyv:canary.valnet.xyz:5600",{"value":"{\"value\":{\"host\":\"canary.valnet.xyz\",\"port\":\"5600\",\"lastPing\":null,\"status\":\"unknown\"},\"expires\":null}"}],["keyv:stable.valnet.xyz:5600",{"value":"{\"value\":{\"host\":\"stable.valnet.xyz\",\"port\":\"5600\",\"lastPing\":null,\"status\":\"unknown\"},\"expires\":null}"}]],"lastExpire":1619217003576}
\ No newline at end of file
diff --git a/yarn.lock b/yarn.lock
index bad6d46..4097dc7 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4550,6 +4550,12 @@ expect@^26.6.2:
     jest-message-util "^26.6.2"
     jest-regex-util "^26.0.0"
 
+express-ws@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/express-ws/-/express-ws-4.0.0.tgz#dabd8dc974516418902a41fe6e30ed949b4d36c4"
+  dependencies:
+    ws "^5.2.0"
+
 express@^4.17.1:
   version "4.17.1"
   resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134"
@@ -10616,6 +10622,12 @@ write@1.0.3:
   dependencies:
     mkdirp "^0.5.1"
 
+ws@^5.2.0:
+  version "5.2.2"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.2.tgz#dffef14866b8e8dc9133582514d1befaf96e980f"
+  dependencies:
+    async-limiter "~1.0.0"
+
 ws@^6.2.1:
   version "6.2.1"
   resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.1.tgz#442fdf0a47ed64f59b6a5d8ff130f4748ed524fb"