Reuse restricted engines with fresh invocation state and revocable ports
This commit is contained in:
+216
-110
@@ -1,4 +1,5 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createServer, request as httpRequest } from "node:http";
|
||||
import { mkdtemp, readFile, writeFile, rm, open } from "node:fs/promises";
|
||||
import { constants } from "node:fs";
|
||||
@@ -34,7 +35,8 @@ async function boundedSource(path, maximum, code) {
|
||||
|
||||
/** Trusted stdio adapter inside one workspace VM. Engine processes never own
|
||||
* coordinator credentials. The supervising service must impose VM/OS limits.
|
||||
* Per-frame process replacement prevents retained globals/handles crossing calls. */
|
||||
* Each engine runs one fresh Worker per invocation. Host tokens expire before
|
||||
* reuse; cancellation and watchdogs destroy the affected engine, not neighbors. */
|
||||
export async function serveRestrictedRegistry(config, io = process) {
|
||||
if (
|
||||
!object(config) ||
|
||||
@@ -68,6 +70,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
maxQueued: config.maxQueued ?? 8,
|
||||
maxActive: config.maxActive ?? 2,
|
||||
maxNested: config.maxNested ?? 4,
|
||||
engineIdleMs: config.engineIdleMs ?? 30000,
|
||||
};
|
||||
if (
|
||||
!Number.isSafeInteger(limits.maxBytes) ||
|
||||
@@ -87,7 +90,10 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
limits.maxActive > 16 ||
|
||||
!Number.isSafeInteger(limits.maxNested) ||
|
||||
limits.maxNested < 1 ||
|
||||
limits.maxNested > 16
|
||||
limits.maxNested > 16 ||
|
||||
!Number.isSafeInteger(limits.engineIdleMs) ||
|
||||
limits.engineIdleMs < 1 ||
|
||||
limits.engineIdleMs > 60000
|
||||
)
|
||||
throw fixedError("INVALID_RESTRICTED_LIMITS");
|
||||
const packageSource = await boundedSource(config.module, 1024 * 1024, "RESTRICTED_SOURCE_LIMIT");
|
||||
@@ -95,7 +101,10 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
const sdk = [];
|
||||
for (const m of config.sdk)
|
||||
sdk.push({ ...m, source: await boundedSource(m.path, 2 * 1024 * 1024, "RESTRICTED_SDK_LIMIT") });
|
||||
const active = new Map();
|
||||
const active = new Map(),
|
||||
engines = new Set(),
|
||||
idleEngines = [];
|
||||
const metrics = { starts: 0, reused: 0, completed: 0, peakRSSBytes: 0 };
|
||||
let buffered = Buffer.alloc(0),
|
||||
stopped = false;
|
||||
const queued = [];
|
||||
@@ -121,7 +130,9 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
throw error;
|
||||
}
|
||||
const fields = text.slice(text.lastIndexOf(")") + 2).split(" ");
|
||||
const cpuMicroseconds = (Number(fields[11]) + Number(fields[12])) * 10000;
|
||||
const total = (Number(fields[11]) + Number(fields[12])) * 10000;
|
||||
const cpuMicroseconds = total - (call.cpuBaseline ?? 0);
|
||||
if (call.engine) call.engine.cpuTotal = total;
|
||||
if (!Number.isSafeInteger(cpuMicroseconds) || cpuMicroseconds < (call.cpuMicroseconds ?? 0))
|
||||
throw fixedError("RESTRICTED_CPU_ACCOUNTING");
|
||||
if (cpuMicroseconds !== (call.cpuMicroseconds ?? 0)) {
|
||||
@@ -158,17 +169,23 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
} catch (error) {
|
||||
usageFailure = error;
|
||||
}
|
||||
if (call.child?.pid) {
|
||||
try {
|
||||
process.kill(-call.child.pid, "SIGKILL");
|
||||
} catch (error) {
|
||||
if (error.code !== "ESRCH") throw error;
|
||||
}
|
||||
await call.exited;
|
||||
const engine = call.engine;
|
||||
if (engine) {
|
||||
// Revoke the frame token before returning an engine to the idle pool.
|
||||
// A delayed child request can never acquire the next call's authority.
|
||||
if (engine.current === call) engine.current = null;
|
||||
if (!stopped && reason.code === "INVOCATION_ENDED" && !usageFailure && !engine.destroying) {
|
||||
const status = await readFile(`/proc/${engine.child.pid}/status`, "utf8").catch(() => "");
|
||||
const rss = Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? Infinity) * 1024;
|
||||
metrics.peakRSSBytes = Math.max(metrics.peakRSSBytes, rss);
|
||||
// Bound retained engine heaps as well as lifetime. Hard aggregate
|
||||
// physical memory remains the supervising process/cgroup's contract.
|
||||
if (rss <= 128 * 1024 * 1024) {
|
||||
idleEngines.push(engine);
|
||||
engine.idleTimer = setTimeout(() => void destroyEngine(engine).catch(fail), limits.engineIdleMs);
|
||||
} else await destroyEngine(engine);
|
||||
} else await destroyEngine(engine);
|
||||
}
|
||||
call.server?.closeAllConnections();
|
||||
if (call.server) await new Promise((resolve) => call.server.close(resolve));
|
||||
if (call.directory) await rm(call.directory, { recursive: true, force: true });
|
||||
active.delete(call.message.invocation);
|
||||
if (usageFailure) throw usageFailure;
|
||||
})();
|
||||
@@ -180,6 +197,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
io.stdin.destroy();
|
||||
for (const call of [...queued]) await retire(call, error);
|
||||
await Promise.all([...active.values()].map((call) => retire(call, error)));
|
||||
await Promise.all([...engines].map(destroyEngine));
|
||||
io.stdout.end();
|
||||
io.exitCode = 1;
|
||||
};
|
||||
@@ -193,6 +211,148 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
}
|
||||
return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks)));
|
||||
};
|
||||
const destroyEngine = async (engine) => {
|
||||
if (engine.destroying) return engine.destroying;
|
||||
engine.destroying = (async () => {
|
||||
clearTimeout(engine.idleTimer);
|
||||
const index = idleEngines.indexOf(engine);
|
||||
if (index >= 0) idleEngines.splice(index, 1);
|
||||
engine.current = null;
|
||||
if (engine.child?.pid) {
|
||||
try {
|
||||
process.kill(-engine.child.pid, "SIGKILL");
|
||||
} catch (error) {
|
||||
if (error.code !== "ESRCH") throw error;
|
||||
}
|
||||
await engine.exited;
|
||||
}
|
||||
engine.server?.closeAllConnections();
|
||||
if (engine.server) await new Promise((resolve) => engine.server.close(resolve));
|
||||
if (engine.directory) await rm(engine.directory, { recursive: true, force: true });
|
||||
engines.delete(engine);
|
||||
})();
|
||||
return engine.destroying;
|
||||
};
|
||||
const prepareEngine = async (call) => {
|
||||
const cached = idleEngines.pop();
|
||||
if (cached) {
|
||||
clearTimeout(cached.idleTimer);
|
||||
cached.current = call;
|
||||
call.engine = cached;
|
||||
call.child = cached.child;
|
||||
call.cpuBaseline = cached.cpuTotal ?? 0;
|
||||
metrics.reused++;
|
||||
watchCPU(call);
|
||||
return;
|
||||
}
|
||||
const engine = { current: call, cpuTotal: 0 };
|
||||
engines.add(engine);
|
||||
call.engine = engine;
|
||||
engine.directory = await mkdtemp(join(tmpdir(), "qx-engine-"));
|
||||
if (!call.open) return;
|
||||
const broker = join(engine.directory, "broker.sock");
|
||||
engine.socket = join(engine.directory, "engine.sock");
|
||||
engine.server = createServer(async (request, response) => {
|
||||
const call = engine.current;
|
||||
const message = call?.message;
|
||||
try {
|
||||
if (!call?.open || request.method !== "POST" || request.url !== "/port/" + call.token)
|
||||
throw fixedError("EXPIRED_CONTEXT");
|
||||
const port = await body(request);
|
||||
if (
|
||||
!call.open ||
|
||||
!object(port) ||
|
||||
Object.keys(port).some((key) => !["portId", "action", "input", "key"].includes(key)) ||
|
||||
typeof port.portId !== "string" ||
|
||||
!object(port.action) ||
|
||||
!Object.hasOwn(port, "input")
|
||||
)
|
||||
throw fixedError("INVALID_RESTRICTED_PORT");
|
||||
if (
|
||||
++call.sequence > Math.min(limits.maxPortCalls, message.budget?.calls ?? limits.maxPortCalls) ||
|
||||
call.pending.size >= limits.maxPortCalls
|
||||
)
|
||||
throw fixedError("RESTRICTED_PORT_LIMIT");
|
||||
const sequence = call.sequence;
|
||||
const result = new Promise((resolve, reject) => call.pending.set(sequence, { resolve, reject }));
|
||||
void result.catch(() => {});
|
||||
send({
|
||||
kind: "port",
|
||||
invocation: message.invocation,
|
||||
frame: message.frame,
|
||||
sequence,
|
||||
portId: port.portId,
|
||||
action: { ...port.action, ...(port.key === undefined ? {} : { key: port.key }) },
|
||||
input: port.input,
|
||||
});
|
||||
const value = await result;
|
||||
if (!call.open) throw fixedError("EXPIRED_CONTEXT");
|
||||
const encoded = JSON.stringify(value);
|
||||
if (Buffer.byteLength(encoded) > limits.maxBytes) throw fixedError("RESTRICTED_MESSAGE_LIMIT");
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(encoded);
|
||||
} catch {
|
||||
response.writeHead(409);
|
||||
response.end('{"status":"failed","error":{"code":"RESTRICTED_PORT_FAILED"}}');
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
engine.server.once("error", reject);
|
||||
engine.server.listen(broker, resolve);
|
||||
});
|
||||
const modules = { "entry.mjs": { js: workerSource }, "package.mjs": { js: packageSource } };
|
||||
for (const module of sdk) modules[module.name] = { js: module.source };
|
||||
const setting = { registryExport: config.registryExport, exports: config.exports, maxBytes: limits.maxBytes };
|
||||
// The parent contains immutable checked source only. It never imports the
|
||||
// authored module or keeps a child stub, request, result or frame handle.
|
||||
const parent = `const modules=${JSON.stringify(modules)}, setting=${JSON.stringify(setting)};
|
||||
export default {async fetch(request,env) {
|
||||
const input=await request.json();
|
||||
const child=env.LOADER.load({compatibilityDate:"2026-10-05",mainModule:"entry.mjs",modules,
|
||||
globalOutbound:null,env:{BROKER:env.BROKER,CONFIG:{...setting,frameToken:input.frameToken}}});
|
||||
const response=await child.getEntrypoint().fetch(new Request("http://invocation/",{
|
||||
method:"POST",body:JSON.stringify(input.message)}));
|
||||
const body=await response.arrayBuffer();
|
||||
return new Response(body,{status:response.status,headers:response.headers});
|
||||
}};`;
|
||||
await writeFile(join(engine.directory, "parent.mjs"), parent);
|
||||
const configuration = `using Workerd = import "/workerd/workerd.capnp"; const config :Workerd.Config = (
|
||||
services=[(name="main",worker=(modules=[(name="parent.mjs",esModule=embed "parent.mjs")],
|
||||
compatibilityDate="2026-10-05",compatibilityFlags=["experimental"],globalOutbound="deny",
|
||||
bindings=[(name="LOADER",workerLoader=()),(name="BROKER",service="broker")])),
|
||||
(name="broker",external=(address=${JSON.stringify("unix:" + broker)},http=())),
|
||||
(name="deny",network=(allow=[]))],
|
||||
sockets=[(name="frame",address=${JSON.stringify("unix:" + engine.socket)},http=(),service="main")]);`;
|
||||
const file = join(engine.directory, "config.capnp");
|
||||
await writeFile(file, configuration);
|
||||
if (!call.open) return;
|
||||
engine.child = spawn(config.workerd, ["serve", "--experimental", file], {
|
||||
detached: true,
|
||||
env: { TZ: "UTC", LANG: "C.UTF-8" },
|
||||
cwd: engine.directory,
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
});
|
||||
metrics.starts++;
|
||||
engine.exited = new Promise((resolve) => {
|
||||
engine.child.once("exit", resolve);
|
||||
engine.child.once("error", resolve);
|
||||
});
|
||||
call.child = engine.child;
|
||||
watchCPU(call);
|
||||
let diagnostics = 0;
|
||||
engine.child.stderr.on("data", (chunk) => {
|
||||
if ((diagnostics += chunk.length) > 65536) {
|
||||
if (engine.current?.open) engine.current.terminate("RESTRICTED_DIAGNOSTIC_LIMIT");
|
||||
else void destroyEngine(engine).catch(fail);
|
||||
}
|
||||
});
|
||||
engine.child.once("error", () => engine.current?.terminate("RESTRICTED_START_FAILED"));
|
||||
void engine.exited.then(() => {
|
||||
if (engine.current?.open) engine.current.terminate("RESTRICTED_ENGINE_EXITED");
|
||||
else void destroyEngine(engine).catch(fail);
|
||||
});
|
||||
};
|
||||
const available = (call) =>
|
||||
[...active.values()].filter((entry) => entry.nested === call.nested).length <
|
||||
(call.nested ? limits.maxNested : limits.maxActive);
|
||||
@@ -207,12 +367,21 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
if (reply.kind === "cancelled") call.cancellationReply = reply;
|
||||
return (call.finished ??= (async () => {
|
||||
await retire(call, reason);
|
||||
metrics.lastInvocation = {
|
||||
queueMs: (call.startedAt ?? performance.now()) - call.receivedAt,
|
||||
preparationMs: call.preparationMs ?? 0,
|
||||
roundtripMs: call.roundtripMs ?? 0,
|
||||
initializationMs: call.initializationMs ?? null,
|
||||
executionMs: call.executionMs ?? null,
|
||||
totalMs: performance.now() - call.receivedAt,
|
||||
};
|
||||
send(call.cancellationReply ?? reply);
|
||||
pump();
|
||||
})());
|
||||
};
|
||||
const invoke = async (call) => {
|
||||
const { message } = call;
|
||||
call.startedAt = performance.now();
|
||||
active.set(message.invocation, call);
|
||||
const terminate = (code) => {
|
||||
if (!call.open) return;
|
||||
@@ -224,100 +393,12 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
};
|
||||
call.terminate = terminate;
|
||||
try {
|
||||
let engine;
|
||||
call.preparing = (async () => {
|
||||
call.directory = await mkdtemp(join(tmpdir(), "qx-frame-"));
|
||||
if (!call.open) return;
|
||||
const broker = join(call.directory, "broker.sock");
|
||||
engine = join(call.directory, "engine.sock");
|
||||
call.server = createServer(async (request, response) => {
|
||||
try {
|
||||
if (!call.open || request.method !== "POST" || request.url !== "/port") throw fixedError("EXPIRED_CONTEXT");
|
||||
const port = await body(request);
|
||||
if (
|
||||
!call.open ||
|
||||
!object(port) ||
|
||||
Object.keys(port).some((key) => !["portId", "action", "input", "key"].includes(key)) ||
|
||||
typeof port.portId !== "string" ||
|
||||
!object(port.action) ||
|
||||
!Object.hasOwn(port, "input")
|
||||
)
|
||||
throw fixedError("INVALID_RESTRICTED_PORT");
|
||||
if (
|
||||
++call.sequence > Math.min(limits.maxPortCalls, message.budget?.calls ?? limits.maxPortCalls) ||
|
||||
call.pending.size >= limits.maxPortCalls
|
||||
)
|
||||
throw fixedError("RESTRICTED_PORT_LIMIT");
|
||||
const sequence = call.sequence;
|
||||
const result = new Promise((resolve, reject) => call.pending.set(sequence, { resolve, reject }));
|
||||
void result.catch(() => {});
|
||||
send({
|
||||
kind: "port",
|
||||
invocation: message.invocation,
|
||||
frame: message.frame,
|
||||
sequence,
|
||||
portId: port.portId,
|
||||
action: { ...port.action, ...(port.key === undefined ? {} : { key: port.key }) },
|
||||
input: port.input,
|
||||
});
|
||||
const value = await result;
|
||||
if (!call.open) throw fixedError("EXPIRED_CONTEXT");
|
||||
const encoded = JSON.stringify(value);
|
||||
if (Buffer.byteLength(encoded) > limits.maxBytes) throw fixedError("RESTRICTED_MESSAGE_LIMIT");
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(encoded);
|
||||
} catch {
|
||||
response.writeHead(409);
|
||||
response.end('{"status":"failed","error":{"code":"RESTRICTED_PORT_FAILED"}}');
|
||||
}
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
call.server.once("error", reject);
|
||||
call.server.listen(broker, resolve);
|
||||
});
|
||||
await writeFile(join(call.directory, "package.mjs"), packageSource);
|
||||
await writeFile(join(call.directory, "entry.mjs"), workerSource);
|
||||
const modules = [
|
||||
{ name: "entry.mjs", esModule: 'embed "entry.mjs"' },
|
||||
{ name: "package.mjs", esModule: 'embed "package.mjs"' },
|
||||
];
|
||||
for (const [index, module] of sdk.entries()) {
|
||||
const file = `sdk-${index}.mjs`;
|
||||
await writeFile(join(call.directory, file), module.source);
|
||||
modules.push({ name: module.name, esModule: "embed " + JSON.stringify(file) });
|
||||
}
|
||||
const setting = { registryExport: config.registryExport, exports: config.exports, maxBytes: limits.maxBytes };
|
||||
const configuration = `using Workerd = import "/workerd/workerd.capnp"; const config :Workerd.Config = (
|
||||
services = [(name="main", worker=(modules=[${modules.map((m) => `(name=${JSON.stringify(m.name)}, esModule=${m.esModule})`).join(",")}],
|
||||
compatibilityDate="2026-10-05", globalOutbound="deny",
|
||||
bindings=[(name="BROKER", service="broker"),(name="CONFIG",json=${JSON.stringify(JSON.stringify(setting))})])),
|
||||
(name="broker",external=(address=${JSON.stringify("unix:" + broker)},http=())), (name="deny",network=(allow=[]))],
|
||||
sockets=[(name="frame",address=${JSON.stringify("unix:" + engine)},http=(),service="main")]);`;
|
||||
const file = join(call.directory, "config.capnp");
|
||||
await writeFile(file, configuration);
|
||||
if (!call.open) return;
|
||||
call.child = spawn(config.workerd, ["serve", file], {
|
||||
detached: true,
|
||||
env: { TZ: "UTC", LANG: "C.UTF-8" },
|
||||
cwd: call.directory,
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
});
|
||||
call.exited = new Promise((resolve) => {
|
||||
call.child.once("exit", resolve);
|
||||
call.child.once("error", resolve);
|
||||
});
|
||||
watchCPU(call);
|
||||
let diagnostics = 0;
|
||||
call.child.stderr.on("data", (chunk) => {
|
||||
if ((diagnostics += chunk.length) > 65536) terminate("RESTRICTED_DIAGNOSTIC_LIMIT");
|
||||
});
|
||||
call.child.once("error", () => terminate("RESTRICTED_START_FAILED"));
|
||||
void call.exited.then(() => {
|
||||
if (call.open) terminate("RESTRICTED_ENGINE_EXITED");
|
||||
});
|
||||
})();
|
||||
call.token = randomUUID();
|
||||
call.preparing = prepareEngine(call);
|
||||
await call.preparing;
|
||||
call.preparationMs = performance.now() - call.startedAt;
|
||||
if (!call.open) return;
|
||||
const roundtripAt = performance.now();
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
const attempt = () => {
|
||||
if (!call.open) {
|
||||
@@ -325,10 +406,17 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
return;
|
||||
}
|
||||
const request = httpRequest(
|
||||
{ socketPath: engine, path: "/", method: "POST", agent: false },
|
||||
{ socketPath: call.engine.socket, path: "/", method: "POST", agent: false },
|
||||
async (response) => {
|
||||
try {
|
||||
const result = await body(response);
|
||||
for (const [header, key] of [
|
||||
["x-quixos-initialization-ms", "initializationMs"],
|
||||
["x-quixos-execution-ms", "executionMs"],
|
||||
]) {
|
||||
const value = Number(response.headers[header]);
|
||||
if (Number.isFinite(value) && value >= 0 && value <= limits.deadlineMs) call[key] = value;
|
||||
}
|
||||
if (response.statusCode !== 200) throw fixedError("RESTRICTED_PACKAGE_FAILED");
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
@@ -340,10 +428,11 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
if (call.open && ["ENOENT", "ECONNREFUSED"].includes(error.code)) setTimeout(attempt, 5);
|
||||
else reject(error);
|
||||
});
|
||||
request.end(JSON.stringify(message));
|
||||
request.end(JSON.stringify({ message, frameToken: call.token }));
|
||||
};
|
||||
attempt();
|
||||
});
|
||||
call.roundtripMs = performance.now() - roundtripAt;
|
||||
if (!call.open) return;
|
||||
if (
|
||||
!object(result) ||
|
||||
@@ -367,6 +456,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
error: { code: result.error.code, message: result.error.code },
|
||||
};
|
||||
await finish(call, fixedError("INVOCATION_ENDED"), reply);
|
||||
metrics.completed++;
|
||||
} catch (error) {
|
||||
if (call.open) {
|
||||
const code = /^RESTRICTED_[A-Z_]+$/.test(error.code ?? "") ? error.code : "RESTRICTED_PACKAGE_FAILED";
|
||||
@@ -412,7 +502,14 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
throw fixedError("INVALID_RESTRICTED_BUDGET");
|
||||
if (active.has(message.invocation) || queued.some((call) => call.message.invocation === message.invocation))
|
||||
throw fixedError("RESTRICTED_DUPLICATE_INVOCATION");
|
||||
const call = { message, open: true, pending: new Map(), sequence: 0, nested: (message.budget?.depth ?? 0) > 1 };
|
||||
const call = {
|
||||
message,
|
||||
receivedAt: performance.now(),
|
||||
open: true,
|
||||
pending: new Map(),
|
||||
sequence: 0,
|
||||
nested: (message.budget?.depth ?? 0) > 1,
|
||||
};
|
||||
// Descendants cannot wait behind ancestors that hold all nested capacity.
|
||||
// Reject excess fan-out/depth rather than deadlock an admitted call tree.
|
||||
if (!available(call) && (call.nested || queued.length >= limits.maxQueued)) {
|
||||
@@ -494,7 +591,16 @@ export async function serveRestrictedRegistry(config, io = process) {
|
||||
io.stdin.on("error", (error) => void fail(error));
|
||||
io.stdout.on("error", (error) => void fail(error));
|
||||
send({ kind: "ready", exports: config.exports });
|
||||
return { close: () => fail(fixedError("SERVICE_SHUTDOWN")) };
|
||||
return {
|
||||
close: () => fail(fixedError("SERVICE_SHUTDOWN")),
|
||||
stats: () => ({
|
||||
...metrics,
|
||||
active: active.size,
|
||||
queued: queued.length,
|
||||
engines: engines.size,
|
||||
idle: idleEngines.length,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
if (process.argv[1] === new URL(import.meta.url).pathname) {
|
||||
|
||||
@@ -77,6 +77,7 @@ async function fixture(source, options = {}) {
|
||||
send,
|
||||
start,
|
||||
usage,
|
||||
stats: service.stats,
|
||||
async close() {
|
||||
await service.close();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
@@ -424,3 +425,149 @@ test("queue wait consumes the deadline and cancelled work never reaches its port
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("warm engines create fresh transitive module state, closures and built-ins", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "qx-transitive-"));
|
||||
const helper = join(directory, "helper.mjs");
|
||||
await writeFile(
|
||||
helper,
|
||||
"let value=0, retained; export function touch(channel){const old=retained; retained=channel; return {count:++value, captured:!!old};}",
|
||||
);
|
||||
const f = await fixture(
|
||||
`import {touch} from 'helper.mjs'; let count=0;
|
||||
export const registry={exports:['test'],async invoke(request,channel){
|
||||
const before={object:Object.prototype.retained??null,array:Array.prototype.retained??null,
|
||||
map:Map.prototype.retained??null, global:globalThis.retained??null};
|
||||
Object.prototype.retained=request.input; Array.prototype.retained=request.input;
|
||||
Map.prototype.retained=request.input; globalThis.retained=request.input;
|
||||
return {count:++count, helper:touch(channel), before};
|
||||
}};`,
|
||||
{ sdk: [{ name: "helper.mjs", path: helper }] },
|
||||
);
|
||||
try {
|
||||
for (const value of ["private graph A", "private graph B", "private graph C"]) {
|
||||
f.start(value);
|
||||
assert.deepEqual((await f.receive()).output, {
|
||||
count: 1,
|
||||
helper: { count: 1, captured: false },
|
||||
before: { object: null, array: null, map: null, global: null },
|
||||
});
|
||||
}
|
||||
assert.equal(f.stats().starts, 1, "Qualification must exercise actual engine reuse");
|
||||
assert.equal(f.stats().reused, 2);
|
||||
assert.equal(f.stats().idle, 1);
|
||||
} finally {
|
||||
await f.close();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("an expired frame token cannot borrow a later invocation's broker", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "qx-token-"));
|
||||
const worker = join(directory, "token-test.mjs");
|
||||
// Trusted test entrypoint deliberately replays the old token. Authored
|
||||
// registries never receive CONFIG or BROKER directly.
|
||||
await writeFile(
|
||||
worker,
|
||||
`export default {async fetch(request,env) {
|
||||
const message=await request.json();
|
||||
const output=message.input ? (await env.BROKER.fetch('http://frame/port/'+message.input,{
|
||||
method:'POST',body:JSON.stringify({portId:'read',action:{kind:'state',primitive:'read'},input:null})})).status
|
||||
: env.CONFIG.frameToken;
|
||||
return Response.json({status:'returned',output});
|
||||
}};`,
|
||||
);
|
||||
const f = await fixture(registry("return null;"), { worker });
|
||||
try {
|
||||
f.start();
|
||||
const token = (await f.receive()).output;
|
||||
assert.match(token, /^[a-f0-9-]{36}$/);
|
||||
f.start(token);
|
||||
assert.equal((await f.receive()).output, 409);
|
||||
assert.equal(f.stats().starts, 1);
|
||||
assert.equal(f.stats().reused, 1);
|
||||
} finally {
|
||||
await f.close();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("idle engines evict without a graph poll and restart with fresh state", async () => {
|
||||
const f = await fixture(registry("return request.input;"), { engineIdleMs: 30 });
|
||||
try {
|
||||
f.start("first");
|
||||
assert.equal((await f.receive()).output, "first");
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
assert.equal(f.stats().engines, 0);
|
||||
f.start("second");
|
||||
assert.equal((await f.receive()).output, "second");
|
||||
assert.equal(f.stats().starts, 2);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
|
||||
for (const loop of [
|
||||
"while(true) await Promise.resolve();",
|
||||
"const memory=[]; for(let n=0;n<128;n++) memory.push(new Uint8Array(256*1024)); while(true) await Promise.resolve(memory);",
|
||||
])
|
||||
test("await/allocation pressure is fenced without delaying a neighboring engine: " + loop.slice(0, 28), async () => {
|
||||
const f = await fixture(registry(`if(request.input==='spin'){${loop}} return request.input;`), {
|
||||
deadlineMs: 1500,
|
||||
});
|
||||
try {
|
||||
const stuck = f.start("spin", { root: "stuck", depth: 1, deadlineMs: 350, calls: 10 });
|
||||
const before = performance.now();
|
||||
const healthy = f.start("healthy");
|
||||
const replies = [await f.receive(), await f.receive()];
|
||||
assert.equal(replies.find((r) => r.invocation === healthy.invocation).output, "healthy");
|
||||
assert.equal(replies.find((r) => r.invocation === stuck.invocation).error.code, "RESTRICTED_DEADLINE");
|
||||
assert.equal(replies[0].invocation, healthy.invocation, "Healthy work must finish before the stuck deadline");
|
||||
assert.ok(performance.now() - before < 1200);
|
||||
f.start("after fault");
|
||||
assert.equal((await f.receive()).output, "after fault");
|
||||
assert.equal(f.stats().starts, 2, "The healthy engine survives its neighbor's termination");
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("detached microtask loops cannot escape the invocation wall deadline", async () => {
|
||||
const f = await fixture(
|
||||
registry("void(async()=>{while(true)await Promise.resolve();})(); return 'nominal return';"),
|
||||
{ deadlineMs: 200 },
|
||||
);
|
||||
try {
|
||||
const before = performance.now();
|
||||
f.start();
|
||||
assert.equal((await f.receive()).error.code, "RESTRICTED_DEADLINE");
|
||||
assert.ok(performance.now() - before < 1000);
|
||||
assert.equal(f.stats().engines, 0);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("detached digest callbacks cannot use a completed frame or contaminate reuse", async () => {
|
||||
const f = await fixture(
|
||||
registry(`
|
||||
void(async()=>{
|
||||
for(let n=0;n<100;n++)await crypto.subtle.digest('SHA-256',new Uint8Array(32768));
|
||||
await channel({portId:'late',action:{kind:'state',primitive:'read'},input:request.input});
|
||||
})().catch(()=>{});
|
||||
return request.input;
|
||||
`),
|
||||
);
|
||||
try {
|
||||
f.start("private A");
|
||||
assert.equal((await f.receive()).output, "private A");
|
||||
await new Promise((resolve) => setTimeout(resolve, 75));
|
||||
f.start("private B");
|
||||
const reply = await f.receive();
|
||||
assert.equal(reply.status, "returned", "No late graph port may appear after completion");
|
||||
assert.equal(reply.output, "private B");
|
||||
assert.equal(f.stats().starts, 1);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// Platform entrypoint, evaluated before loading the authored registry. A fresh
|
||||
// engine process owns exactly one frame. BROKER grants only that frame's ports.
|
||||
// Worker owns exactly one frame. The parent engine is reusable; BROKER's host
|
||||
// token grants only this frame and expires before the next invocation.
|
||||
const monotonic = performance.now.bind(performance);
|
||||
const deny = () => {
|
||||
throw Error("RESTRICTED_AMBIENT_ACCESS");
|
||||
};
|
||||
@@ -48,7 +50,9 @@ export default {
|
||||
if (used) return new Response("FRAME_EXPIRED", { status: 409 });
|
||||
used = true;
|
||||
const input = await request.json();
|
||||
const loading = monotonic();
|
||||
const source = await import("./package.mjs");
|
||||
const initializationMs = monotonic() - loading;
|
||||
const registry = source[env.CONFIG.registryExport];
|
||||
if (
|
||||
!registry ||
|
||||
@@ -59,7 +63,7 @@ export default {
|
||||
let open = true;
|
||||
const channel = async (value) => {
|
||||
if (!open) throw Error("EXPIRED_CONTEXT");
|
||||
const response = await env.BROKER.fetch("http://frame/port", {
|
||||
const response = await env.BROKER.fetch("http://frame/port/" + env.CONFIG.frameToken, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(value),
|
||||
});
|
||||
@@ -71,6 +75,7 @@ export default {
|
||||
});
|
||||
return result.output;
|
||||
};
|
||||
const running = monotonic();
|
||||
try {
|
||||
const output = await registry.invoke(
|
||||
{ exportId: input.exportId, workspace: input.workspace, receiver: input.receiver, input: input.input },
|
||||
@@ -78,7 +83,13 @@ export default {
|
||||
);
|
||||
const body = JSON.stringify({ status: "returned", output });
|
||||
if (new TextEncoder().encode(body).length > env.CONFIG.maxBytes) throw Error("RESTRICTED_OUTPUT_LIMIT");
|
||||
return new Response(body, { headers: { "content-type": "application/json" } });
|
||||
return new Response(body, {
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-quixos-initialization-ms": String(initializationMs),
|
||||
"x-quixos-execution-ms": String(monotonic() - running),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
// Only authored/codec rejection codes cross the engine boundary. Engine
|
||||
// diagnostics stay private to the supervisor, never become protocol data.
|
||||
|
||||
Reference in New Issue
Block a user