diff --git a/.quixos-subtree-source.json b/.quixos-subtree-source.json index d3b276e..3b8ebb3 100644 --- a/.quixos-subtree-source.json +++ b/.quixos-subtree-source.json @@ -1,7 +1,7 @@ { "version": 1, "sourceRepo": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos", - "sourceCommit": "57684c7548883dc4fc25dc8f943c7a43b3489fcf", + "sourceCommit": "806ea0e9f41340acf119b7374f5150ef5b378074", "sourcePath": "quixos-instance/quixos-nix-helpers", "exportName": "quixos-nix-helpers", "mirrorRemote": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git" diff --git a/restricted-execution/service.mjs b/restricted-execution/service.mjs index c8efc7d..3db0ade 100644 --- a/restricted-execution/service.mjs +++ b/restricted-execution/service.mjs @@ -148,6 +148,11 @@ export async function serveRestrictedRegistry(config, io = process) { engines = new Set(), idleEngines = []; const metrics = { starts: 0, reused: 0, completed: 0, peakRSSBytes: 0 }; + const changed = () => { + try { + config.onChange?.(); + } catch {} + }; let buffered = Buffer.alloc(0), stopped = false, failure; @@ -244,6 +249,7 @@ export async function serveRestrictedRegistry(config, io = process) { // physical memory remains the supervising process/cgroup's contract. if (rss <= 128 * 1024 * 1024) { idleEngines.push(engine); + engine.idleUntil = Date.now() + limits.engineIdleMs; engine.idleTimer = setTimeout(() => void destroyEngine(engine).catch(fail), limits.engineIdleMs); } else await destroyEngine(engine); } else await destroyEngine(engine); @@ -300,6 +306,7 @@ export async function serveRestrictedRegistry(config, io = process) { } finally { engines.delete(engine); await engine.release?.(); + changed(); } })(); return engine.destroying; @@ -418,6 +425,7 @@ export async function serveRestrictedRegistry(config, io = process) { }, ); metrics.starts++; + changed(); engine.exited = new Promise((resolve) => { engine.child.once("exit", resolve); engine.child.once("error", resolve); @@ -462,6 +470,9 @@ export async function serveRestrictedRegistry(config, io = process) { executionMs: call.executionMs ?? null, totalMs: performance.now() - call.receivedAt, }; + metrics.lastFailure = + reply.status === "failed" && /^[A-Z0-9_]{1,128}$/.test(reply.error?.code) ? reply.error.code : null; + changed(); send(call.cancellationReply ?? reply); pump(); })()); @@ -470,6 +481,7 @@ export async function serveRestrictedRegistry(config, io = process) { const { message } = call; call.startedAt = performance.now(); active.set(message.invocation, call); + changed(); const terminate = (code) => { if (!call.open) return; void finish(call, fixedError(code), { @@ -687,6 +699,9 @@ export async function serveRestrictedRegistry(config, io = process) { queued: queued.length, engines: engines.size, idle: idleEngines.length, + idleDeadline: idleEngines.length + ? new Date(Math.min(...idleEngines.map((engine) => engine.idleUntil))).toISOString() + : null, }), }; } diff --git a/restricted-execution/service.test.mjs b/restricted-execution/service.test.mjs index b6087b4..409f615 100644 --- a/restricted-execution/service.test.mjs +++ b/restricted-execution/service.test.mjs @@ -570,12 +570,25 @@ test("an expired frame token cannot borrow a later invocation's broker", async ( }); test("idle engines evict without a graph poll and restart with fresh state", async () => { - const f = await fixture(registry("return request.input;"), { engineIdleMs: 30 }); + const updates = []; + const f = await fixture(registry("return request.input;"), { + engineIdleMs: 30, + onChange: () => updates.push(f.stats()), + }); try { f.start("first"); assert.equal((await f.receive()).output, "first"); await new Promise((resolve) => setTimeout(resolve, 100)); assert.equal(f.stats().engines, 0); + assert.ok( + updates.some((s) => s.active > 0), + "Inspectors observe active invocation admission", + ); + assert.ok( + updates.some((s) => s.idle > 0 && s.idleDeadline), + "Idle eviction deadline is visible", + ); + assert.equal(updates.at(-1).engines, 0, "Eviction pushes an update without a graph poll"); f.start("second"); assert.equal((await f.receive()).output, "second"); assert.equal(f.stats().starts, 2);