Reuse restricted engines with fresh invocation state and revocable ports
This commit is contained in:
+216
-110
@@ -1,4 +1,5 @@
|
|||||||
import { spawn } from "node:child_process";
|
import { spawn } from "node:child_process";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
import { createServer, request as httpRequest } from "node:http";
|
import { createServer, request as httpRequest } from "node:http";
|
||||||
import { mkdtemp, readFile, writeFile, rm, open } from "node:fs/promises";
|
import { mkdtemp, readFile, writeFile, rm, open } from "node:fs/promises";
|
||||||
import { constants } from "node:fs";
|
import { constants } from "node:fs";
|
||||||
@@ -34,7 +35,8 @@ async function boundedSource(path, maximum, code) {
|
|||||||
|
|
||||||
/** Trusted stdio adapter inside one workspace VM. Engine processes never own
|
/** Trusted stdio adapter inside one workspace VM. Engine processes never own
|
||||||
* coordinator credentials. The supervising service must impose VM/OS limits.
|
* coordinator credentials. The supervising service must impose VM/OS limits.
|
||||||
* Per-frame process replacement prevents retained globals/handles crossing calls. */
|
* Each engine runs one fresh Worker per invocation. Host tokens expire before
|
||||||
|
* reuse; cancellation and watchdogs destroy the affected engine, not neighbors. */
|
||||||
export async function serveRestrictedRegistry(config, io = process) {
|
export async function serveRestrictedRegistry(config, io = process) {
|
||||||
if (
|
if (
|
||||||
!object(config) ||
|
!object(config) ||
|
||||||
@@ -68,6 +70,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
maxQueued: config.maxQueued ?? 8,
|
maxQueued: config.maxQueued ?? 8,
|
||||||
maxActive: config.maxActive ?? 2,
|
maxActive: config.maxActive ?? 2,
|
||||||
maxNested: config.maxNested ?? 4,
|
maxNested: config.maxNested ?? 4,
|
||||||
|
engineIdleMs: config.engineIdleMs ?? 30000,
|
||||||
};
|
};
|
||||||
if (
|
if (
|
||||||
!Number.isSafeInteger(limits.maxBytes) ||
|
!Number.isSafeInteger(limits.maxBytes) ||
|
||||||
@@ -87,7 +90,10 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
limits.maxActive > 16 ||
|
limits.maxActive > 16 ||
|
||||||
!Number.isSafeInteger(limits.maxNested) ||
|
!Number.isSafeInteger(limits.maxNested) ||
|
||||||
limits.maxNested < 1 ||
|
limits.maxNested < 1 ||
|
||||||
limits.maxNested > 16
|
limits.maxNested > 16 ||
|
||||||
|
!Number.isSafeInteger(limits.engineIdleMs) ||
|
||||||
|
limits.engineIdleMs < 1 ||
|
||||||
|
limits.engineIdleMs > 60000
|
||||||
)
|
)
|
||||||
throw fixedError("INVALID_RESTRICTED_LIMITS");
|
throw fixedError("INVALID_RESTRICTED_LIMITS");
|
||||||
const packageSource = await boundedSource(config.module, 1024 * 1024, "RESTRICTED_SOURCE_LIMIT");
|
const packageSource = await boundedSource(config.module, 1024 * 1024, "RESTRICTED_SOURCE_LIMIT");
|
||||||
@@ -95,7 +101,10 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
const sdk = [];
|
const sdk = [];
|
||||||
for (const m of config.sdk)
|
for (const m of config.sdk)
|
||||||
sdk.push({ ...m, source: await boundedSource(m.path, 2 * 1024 * 1024, "RESTRICTED_SDK_LIMIT") });
|
sdk.push({ ...m, source: await boundedSource(m.path, 2 * 1024 * 1024, "RESTRICTED_SDK_LIMIT") });
|
||||||
const active = new Map();
|
const active = new Map(),
|
||||||
|
engines = new Set(),
|
||||||
|
idleEngines = [];
|
||||||
|
const metrics = { starts: 0, reused: 0, completed: 0, peakRSSBytes: 0 };
|
||||||
let buffered = Buffer.alloc(0),
|
let buffered = Buffer.alloc(0),
|
||||||
stopped = false;
|
stopped = false;
|
||||||
const queued = [];
|
const queued = [];
|
||||||
@@ -121,7 +130,9 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
const fields = text.slice(text.lastIndexOf(")") + 2).split(" ");
|
const fields = text.slice(text.lastIndexOf(")") + 2).split(" ");
|
||||||
const cpuMicroseconds = (Number(fields[11]) + Number(fields[12])) * 10000;
|
const total = (Number(fields[11]) + Number(fields[12])) * 10000;
|
||||||
|
const cpuMicroseconds = total - (call.cpuBaseline ?? 0);
|
||||||
|
if (call.engine) call.engine.cpuTotal = total;
|
||||||
if (!Number.isSafeInteger(cpuMicroseconds) || cpuMicroseconds < (call.cpuMicroseconds ?? 0))
|
if (!Number.isSafeInteger(cpuMicroseconds) || cpuMicroseconds < (call.cpuMicroseconds ?? 0))
|
||||||
throw fixedError("RESTRICTED_CPU_ACCOUNTING");
|
throw fixedError("RESTRICTED_CPU_ACCOUNTING");
|
||||||
if (cpuMicroseconds !== (call.cpuMicroseconds ?? 0)) {
|
if (cpuMicroseconds !== (call.cpuMicroseconds ?? 0)) {
|
||||||
@@ -158,17 +169,23 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
usageFailure = error;
|
usageFailure = error;
|
||||||
}
|
}
|
||||||
if (call.child?.pid) {
|
const engine = call.engine;
|
||||||
try {
|
if (engine) {
|
||||||
process.kill(-call.child.pid, "SIGKILL");
|
// Revoke the frame token before returning an engine to the idle pool.
|
||||||
} catch (error) {
|
// A delayed child request can never acquire the next call's authority.
|
||||||
if (error.code !== "ESRCH") throw error;
|
if (engine.current === call) engine.current = null;
|
||||||
}
|
if (!stopped && reason.code === "INVOCATION_ENDED" && !usageFailure && !engine.destroying) {
|
||||||
await call.exited;
|
const status = await readFile(`/proc/${engine.child.pid}/status`, "utf8").catch(() => "");
|
||||||
|
const rss = Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? Infinity) * 1024;
|
||||||
|
metrics.peakRSSBytes = Math.max(metrics.peakRSSBytes, rss);
|
||||||
|
// Bound retained engine heaps as well as lifetime. Hard aggregate
|
||||||
|
// physical memory remains the supervising process/cgroup's contract.
|
||||||
|
if (rss <= 128 * 1024 * 1024) {
|
||||||
|
idleEngines.push(engine);
|
||||||
|
engine.idleTimer = setTimeout(() => void destroyEngine(engine).catch(fail), limits.engineIdleMs);
|
||||||
|
} else await destroyEngine(engine);
|
||||||
|
} else await destroyEngine(engine);
|
||||||
}
|
}
|
||||||
call.server?.closeAllConnections();
|
|
||||||
if (call.server) await new Promise((resolve) => call.server.close(resolve));
|
|
||||||
if (call.directory) await rm(call.directory, { recursive: true, force: true });
|
|
||||||
active.delete(call.message.invocation);
|
active.delete(call.message.invocation);
|
||||||
if (usageFailure) throw usageFailure;
|
if (usageFailure) throw usageFailure;
|
||||||
})();
|
})();
|
||||||
@@ -180,6 +197,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
io.stdin.destroy();
|
io.stdin.destroy();
|
||||||
for (const call of [...queued]) await retire(call, error);
|
for (const call of [...queued]) await retire(call, error);
|
||||||
await Promise.all([...active.values()].map((call) => retire(call, error)));
|
await Promise.all([...active.values()].map((call) => retire(call, error)));
|
||||||
|
await Promise.all([...engines].map(destroyEngine));
|
||||||
io.stdout.end();
|
io.stdout.end();
|
||||||
io.exitCode = 1;
|
io.exitCode = 1;
|
||||||
};
|
};
|
||||||
@@ -193,6 +211,148 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
}
|
}
|
||||||
return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks)));
|
return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks)));
|
||||||
};
|
};
|
||||||
|
const destroyEngine = async (engine) => {
|
||||||
|
if (engine.destroying) return engine.destroying;
|
||||||
|
engine.destroying = (async () => {
|
||||||
|
clearTimeout(engine.idleTimer);
|
||||||
|
const index = idleEngines.indexOf(engine);
|
||||||
|
if (index >= 0) idleEngines.splice(index, 1);
|
||||||
|
engine.current = null;
|
||||||
|
if (engine.child?.pid) {
|
||||||
|
try {
|
||||||
|
process.kill(-engine.child.pid, "SIGKILL");
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code !== "ESRCH") throw error;
|
||||||
|
}
|
||||||
|
await engine.exited;
|
||||||
|
}
|
||||||
|
engine.server?.closeAllConnections();
|
||||||
|
if (engine.server) await new Promise((resolve) => engine.server.close(resolve));
|
||||||
|
if (engine.directory) await rm(engine.directory, { recursive: true, force: true });
|
||||||
|
engines.delete(engine);
|
||||||
|
})();
|
||||||
|
return engine.destroying;
|
||||||
|
};
|
||||||
|
const prepareEngine = async (call) => {
|
||||||
|
const cached = idleEngines.pop();
|
||||||
|
if (cached) {
|
||||||
|
clearTimeout(cached.idleTimer);
|
||||||
|
cached.current = call;
|
||||||
|
call.engine = cached;
|
||||||
|
call.child = cached.child;
|
||||||
|
call.cpuBaseline = cached.cpuTotal ?? 0;
|
||||||
|
metrics.reused++;
|
||||||
|
watchCPU(call);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const engine = { current: call, cpuTotal: 0 };
|
||||||
|
engines.add(engine);
|
||||||
|
call.engine = engine;
|
||||||
|
engine.directory = await mkdtemp(join(tmpdir(), "qx-engine-"));
|
||||||
|
if (!call.open) return;
|
||||||
|
const broker = join(engine.directory, "broker.sock");
|
||||||
|
engine.socket = join(engine.directory, "engine.sock");
|
||||||
|
engine.server = createServer(async (request, response) => {
|
||||||
|
const call = engine.current;
|
||||||
|
const message = call?.message;
|
||||||
|
try {
|
||||||
|
if (!call?.open || request.method !== "POST" || request.url !== "/port/" + call.token)
|
||||||
|
throw fixedError("EXPIRED_CONTEXT");
|
||||||
|
const port = await body(request);
|
||||||
|
if (
|
||||||
|
!call.open ||
|
||||||
|
!object(port) ||
|
||||||
|
Object.keys(port).some((key) => !["portId", "action", "input", "key"].includes(key)) ||
|
||||||
|
typeof port.portId !== "string" ||
|
||||||
|
!object(port.action) ||
|
||||||
|
!Object.hasOwn(port, "input")
|
||||||
|
)
|
||||||
|
throw fixedError("INVALID_RESTRICTED_PORT");
|
||||||
|
if (
|
||||||
|
++call.sequence > Math.min(limits.maxPortCalls, message.budget?.calls ?? limits.maxPortCalls) ||
|
||||||
|
call.pending.size >= limits.maxPortCalls
|
||||||
|
)
|
||||||
|
throw fixedError("RESTRICTED_PORT_LIMIT");
|
||||||
|
const sequence = call.sequence;
|
||||||
|
const result = new Promise((resolve, reject) => call.pending.set(sequence, { resolve, reject }));
|
||||||
|
void result.catch(() => {});
|
||||||
|
send({
|
||||||
|
kind: "port",
|
||||||
|
invocation: message.invocation,
|
||||||
|
frame: message.frame,
|
||||||
|
sequence,
|
||||||
|
portId: port.portId,
|
||||||
|
action: { ...port.action, ...(port.key === undefined ? {} : { key: port.key }) },
|
||||||
|
input: port.input,
|
||||||
|
});
|
||||||
|
const value = await result;
|
||||||
|
if (!call.open) throw fixedError("EXPIRED_CONTEXT");
|
||||||
|
const encoded = JSON.stringify(value);
|
||||||
|
if (Buffer.byteLength(encoded) > limits.maxBytes) throw fixedError("RESTRICTED_MESSAGE_LIMIT");
|
||||||
|
response.writeHead(200, { "content-type": "application/json" });
|
||||||
|
response.end(encoded);
|
||||||
|
} catch {
|
||||||
|
response.writeHead(409);
|
||||||
|
response.end('{"status":"failed","error":{"code":"RESTRICTED_PORT_FAILED"}}');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
engine.server.once("error", reject);
|
||||||
|
engine.server.listen(broker, resolve);
|
||||||
|
});
|
||||||
|
const modules = { "entry.mjs": { js: workerSource }, "package.mjs": { js: packageSource } };
|
||||||
|
for (const module of sdk) modules[module.name] = { js: module.source };
|
||||||
|
const setting = { registryExport: config.registryExport, exports: config.exports, maxBytes: limits.maxBytes };
|
||||||
|
// The parent contains immutable checked source only. It never imports the
|
||||||
|
// authored module or keeps a child stub, request, result or frame handle.
|
||||||
|
const parent = `const modules=${JSON.stringify(modules)}, setting=${JSON.stringify(setting)};
|
||||||
|
export default {async fetch(request,env) {
|
||||||
|
const input=await request.json();
|
||||||
|
const child=env.LOADER.load({compatibilityDate:"2026-10-05",mainModule:"entry.mjs",modules,
|
||||||
|
globalOutbound:null,env:{BROKER:env.BROKER,CONFIG:{...setting,frameToken:input.frameToken}}});
|
||||||
|
const response=await child.getEntrypoint().fetch(new Request("http://invocation/",{
|
||||||
|
method:"POST",body:JSON.stringify(input.message)}));
|
||||||
|
const body=await response.arrayBuffer();
|
||||||
|
return new Response(body,{status:response.status,headers:response.headers});
|
||||||
|
}};`;
|
||||||
|
await writeFile(join(engine.directory, "parent.mjs"), parent);
|
||||||
|
const configuration = `using Workerd = import "/workerd/workerd.capnp"; const config :Workerd.Config = (
|
||||||
|
services=[(name="main",worker=(modules=[(name="parent.mjs",esModule=embed "parent.mjs")],
|
||||||
|
compatibilityDate="2026-10-05",compatibilityFlags=["experimental"],globalOutbound="deny",
|
||||||
|
bindings=[(name="LOADER",workerLoader=()),(name="BROKER",service="broker")])),
|
||||||
|
(name="broker",external=(address=${JSON.stringify("unix:" + broker)},http=())),
|
||||||
|
(name="deny",network=(allow=[]))],
|
||||||
|
sockets=[(name="frame",address=${JSON.stringify("unix:" + engine.socket)},http=(),service="main")]);`;
|
||||||
|
const file = join(engine.directory, "config.capnp");
|
||||||
|
await writeFile(file, configuration);
|
||||||
|
if (!call.open) return;
|
||||||
|
engine.child = spawn(config.workerd, ["serve", "--experimental", file], {
|
||||||
|
detached: true,
|
||||||
|
env: { TZ: "UTC", LANG: "C.UTF-8" },
|
||||||
|
cwd: engine.directory,
|
||||||
|
stdio: ["ignore", "ignore", "pipe"],
|
||||||
|
});
|
||||||
|
metrics.starts++;
|
||||||
|
engine.exited = new Promise((resolve) => {
|
||||||
|
engine.child.once("exit", resolve);
|
||||||
|
engine.child.once("error", resolve);
|
||||||
|
});
|
||||||
|
call.child = engine.child;
|
||||||
|
watchCPU(call);
|
||||||
|
let diagnostics = 0;
|
||||||
|
engine.child.stderr.on("data", (chunk) => {
|
||||||
|
if ((diagnostics += chunk.length) > 65536) {
|
||||||
|
if (engine.current?.open) engine.current.terminate("RESTRICTED_DIAGNOSTIC_LIMIT");
|
||||||
|
else void destroyEngine(engine).catch(fail);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
engine.child.once("error", () => engine.current?.terminate("RESTRICTED_START_FAILED"));
|
||||||
|
void engine.exited.then(() => {
|
||||||
|
if (engine.current?.open) engine.current.terminate("RESTRICTED_ENGINE_EXITED");
|
||||||
|
else void destroyEngine(engine).catch(fail);
|
||||||
|
});
|
||||||
|
};
|
||||||
const available = (call) =>
|
const available = (call) =>
|
||||||
[...active.values()].filter((entry) => entry.nested === call.nested).length <
|
[...active.values()].filter((entry) => entry.nested === call.nested).length <
|
||||||
(call.nested ? limits.maxNested : limits.maxActive);
|
(call.nested ? limits.maxNested : limits.maxActive);
|
||||||
@@ -207,12 +367,21 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
if (reply.kind === "cancelled") call.cancellationReply = reply;
|
if (reply.kind === "cancelled") call.cancellationReply = reply;
|
||||||
return (call.finished ??= (async () => {
|
return (call.finished ??= (async () => {
|
||||||
await retire(call, reason);
|
await retire(call, reason);
|
||||||
|
metrics.lastInvocation = {
|
||||||
|
queueMs: (call.startedAt ?? performance.now()) - call.receivedAt,
|
||||||
|
preparationMs: call.preparationMs ?? 0,
|
||||||
|
roundtripMs: call.roundtripMs ?? 0,
|
||||||
|
initializationMs: call.initializationMs ?? null,
|
||||||
|
executionMs: call.executionMs ?? null,
|
||||||
|
totalMs: performance.now() - call.receivedAt,
|
||||||
|
};
|
||||||
send(call.cancellationReply ?? reply);
|
send(call.cancellationReply ?? reply);
|
||||||
pump();
|
pump();
|
||||||
})());
|
})());
|
||||||
};
|
};
|
||||||
const invoke = async (call) => {
|
const invoke = async (call) => {
|
||||||
const { message } = call;
|
const { message } = call;
|
||||||
|
call.startedAt = performance.now();
|
||||||
active.set(message.invocation, call);
|
active.set(message.invocation, call);
|
||||||
const terminate = (code) => {
|
const terminate = (code) => {
|
||||||
if (!call.open) return;
|
if (!call.open) return;
|
||||||
@@ -224,100 +393,12 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
};
|
};
|
||||||
call.terminate = terminate;
|
call.terminate = terminate;
|
||||||
try {
|
try {
|
||||||
let engine;
|
call.token = randomUUID();
|
||||||
call.preparing = (async () => {
|
call.preparing = prepareEngine(call);
|
||||||
call.directory = await mkdtemp(join(tmpdir(), "qx-frame-"));
|
|
||||||
if (!call.open) return;
|
|
||||||
const broker = join(call.directory, "broker.sock");
|
|
||||||
engine = join(call.directory, "engine.sock");
|
|
||||||
call.server = createServer(async (request, response) => {
|
|
||||||
try {
|
|
||||||
if (!call.open || request.method !== "POST" || request.url !== "/port") throw fixedError("EXPIRED_CONTEXT");
|
|
||||||
const port = await body(request);
|
|
||||||
if (
|
|
||||||
!call.open ||
|
|
||||||
!object(port) ||
|
|
||||||
Object.keys(port).some((key) => !["portId", "action", "input", "key"].includes(key)) ||
|
|
||||||
typeof port.portId !== "string" ||
|
|
||||||
!object(port.action) ||
|
|
||||||
!Object.hasOwn(port, "input")
|
|
||||||
)
|
|
||||||
throw fixedError("INVALID_RESTRICTED_PORT");
|
|
||||||
if (
|
|
||||||
++call.sequence > Math.min(limits.maxPortCalls, message.budget?.calls ?? limits.maxPortCalls) ||
|
|
||||||
call.pending.size >= limits.maxPortCalls
|
|
||||||
)
|
|
||||||
throw fixedError("RESTRICTED_PORT_LIMIT");
|
|
||||||
const sequence = call.sequence;
|
|
||||||
const result = new Promise((resolve, reject) => call.pending.set(sequence, { resolve, reject }));
|
|
||||||
void result.catch(() => {});
|
|
||||||
send({
|
|
||||||
kind: "port",
|
|
||||||
invocation: message.invocation,
|
|
||||||
frame: message.frame,
|
|
||||||
sequence,
|
|
||||||
portId: port.portId,
|
|
||||||
action: { ...port.action, ...(port.key === undefined ? {} : { key: port.key }) },
|
|
||||||
input: port.input,
|
|
||||||
});
|
|
||||||
const value = await result;
|
|
||||||
if (!call.open) throw fixedError("EXPIRED_CONTEXT");
|
|
||||||
const encoded = JSON.stringify(value);
|
|
||||||
if (Buffer.byteLength(encoded) > limits.maxBytes) throw fixedError("RESTRICTED_MESSAGE_LIMIT");
|
|
||||||
response.writeHead(200, { "content-type": "application/json" });
|
|
||||||
response.end(encoded);
|
|
||||||
} catch {
|
|
||||||
response.writeHead(409);
|
|
||||||
response.end('{"status":"failed","error":{"code":"RESTRICTED_PORT_FAILED"}}');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await new Promise((resolve, reject) => {
|
|
||||||
call.server.once("error", reject);
|
|
||||||
call.server.listen(broker, resolve);
|
|
||||||
});
|
|
||||||
await writeFile(join(call.directory, "package.mjs"), packageSource);
|
|
||||||
await writeFile(join(call.directory, "entry.mjs"), workerSource);
|
|
||||||
const modules = [
|
|
||||||
{ name: "entry.mjs", esModule: 'embed "entry.mjs"' },
|
|
||||||
{ name: "package.mjs", esModule: 'embed "package.mjs"' },
|
|
||||||
];
|
|
||||||
for (const [index, module] of sdk.entries()) {
|
|
||||||
const file = `sdk-${index}.mjs`;
|
|
||||||
await writeFile(join(call.directory, file), module.source);
|
|
||||||
modules.push({ name: module.name, esModule: "embed " + JSON.stringify(file) });
|
|
||||||
}
|
|
||||||
const setting = { registryExport: config.registryExport, exports: config.exports, maxBytes: limits.maxBytes };
|
|
||||||
const configuration = `using Workerd = import "/workerd/workerd.capnp"; const config :Workerd.Config = (
|
|
||||||
services = [(name="main", worker=(modules=[${modules.map((m) => `(name=${JSON.stringify(m.name)}, esModule=${m.esModule})`).join(",")}],
|
|
||||||
compatibilityDate="2026-10-05", globalOutbound="deny",
|
|
||||||
bindings=[(name="BROKER", service="broker"),(name="CONFIG",json=${JSON.stringify(JSON.stringify(setting))})])),
|
|
||||||
(name="broker",external=(address=${JSON.stringify("unix:" + broker)},http=())), (name="deny",network=(allow=[]))],
|
|
||||||
sockets=[(name="frame",address=${JSON.stringify("unix:" + engine)},http=(),service="main")]);`;
|
|
||||||
const file = join(call.directory, "config.capnp");
|
|
||||||
await writeFile(file, configuration);
|
|
||||||
if (!call.open) return;
|
|
||||||
call.child = spawn(config.workerd, ["serve", file], {
|
|
||||||
detached: true,
|
|
||||||
env: { TZ: "UTC", LANG: "C.UTF-8" },
|
|
||||||
cwd: call.directory,
|
|
||||||
stdio: ["ignore", "ignore", "pipe"],
|
|
||||||
});
|
|
||||||
call.exited = new Promise((resolve) => {
|
|
||||||
call.child.once("exit", resolve);
|
|
||||||
call.child.once("error", resolve);
|
|
||||||
});
|
|
||||||
watchCPU(call);
|
|
||||||
let diagnostics = 0;
|
|
||||||
call.child.stderr.on("data", (chunk) => {
|
|
||||||
if ((diagnostics += chunk.length) > 65536) terminate("RESTRICTED_DIAGNOSTIC_LIMIT");
|
|
||||||
});
|
|
||||||
call.child.once("error", () => terminate("RESTRICTED_START_FAILED"));
|
|
||||||
void call.exited.then(() => {
|
|
||||||
if (call.open) terminate("RESTRICTED_ENGINE_EXITED");
|
|
||||||
});
|
|
||||||
})();
|
|
||||||
await call.preparing;
|
await call.preparing;
|
||||||
|
call.preparationMs = performance.now() - call.startedAt;
|
||||||
if (!call.open) return;
|
if (!call.open) return;
|
||||||
|
const roundtripAt = performance.now();
|
||||||
const result = await new Promise((resolve, reject) => {
|
const result = await new Promise((resolve, reject) => {
|
||||||
const attempt = () => {
|
const attempt = () => {
|
||||||
if (!call.open) {
|
if (!call.open) {
|
||||||
@@ -325,10 +406,17 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const request = httpRequest(
|
const request = httpRequest(
|
||||||
{ socketPath: engine, path: "/", method: "POST", agent: false },
|
{ socketPath: call.engine.socket, path: "/", method: "POST", agent: false },
|
||||||
async (response) => {
|
async (response) => {
|
||||||
try {
|
try {
|
||||||
const result = await body(response);
|
const result = await body(response);
|
||||||
|
for (const [header, key] of [
|
||||||
|
["x-quixos-initialization-ms", "initializationMs"],
|
||||||
|
["x-quixos-execution-ms", "executionMs"],
|
||||||
|
]) {
|
||||||
|
const value = Number(response.headers[header]);
|
||||||
|
if (Number.isFinite(value) && value >= 0 && value <= limits.deadlineMs) call[key] = value;
|
||||||
|
}
|
||||||
if (response.statusCode !== 200) throw fixedError("RESTRICTED_PACKAGE_FAILED");
|
if (response.statusCode !== 200) throw fixedError("RESTRICTED_PACKAGE_FAILED");
|
||||||
resolve(result);
|
resolve(result);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -340,10 +428,11 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
if (call.open && ["ENOENT", "ECONNREFUSED"].includes(error.code)) setTimeout(attempt, 5);
|
if (call.open && ["ENOENT", "ECONNREFUSED"].includes(error.code)) setTimeout(attempt, 5);
|
||||||
else reject(error);
|
else reject(error);
|
||||||
});
|
});
|
||||||
request.end(JSON.stringify(message));
|
request.end(JSON.stringify({ message, frameToken: call.token }));
|
||||||
};
|
};
|
||||||
attempt();
|
attempt();
|
||||||
});
|
});
|
||||||
|
call.roundtripMs = performance.now() - roundtripAt;
|
||||||
if (!call.open) return;
|
if (!call.open) return;
|
||||||
if (
|
if (
|
||||||
!object(result) ||
|
!object(result) ||
|
||||||
@@ -367,6 +456,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
error: { code: result.error.code, message: result.error.code },
|
error: { code: result.error.code, message: result.error.code },
|
||||||
};
|
};
|
||||||
await finish(call, fixedError("INVOCATION_ENDED"), reply);
|
await finish(call, fixedError("INVOCATION_ENDED"), reply);
|
||||||
|
metrics.completed++;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (call.open) {
|
if (call.open) {
|
||||||
const code = /^RESTRICTED_[A-Z_]+$/.test(error.code ?? "") ? error.code : "RESTRICTED_PACKAGE_FAILED";
|
const code = /^RESTRICTED_[A-Z_]+$/.test(error.code ?? "") ? error.code : "RESTRICTED_PACKAGE_FAILED";
|
||||||
@@ -412,7 +502,14 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
throw fixedError("INVALID_RESTRICTED_BUDGET");
|
throw fixedError("INVALID_RESTRICTED_BUDGET");
|
||||||
if (active.has(message.invocation) || queued.some((call) => call.message.invocation === message.invocation))
|
if (active.has(message.invocation) || queued.some((call) => call.message.invocation === message.invocation))
|
||||||
throw fixedError("RESTRICTED_DUPLICATE_INVOCATION");
|
throw fixedError("RESTRICTED_DUPLICATE_INVOCATION");
|
||||||
const call = { message, open: true, pending: new Map(), sequence: 0, nested: (message.budget?.depth ?? 0) > 1 };
|
const call = {
|
||||||
|
message,
|
||||||
|
receivedAt: performance.now(),
|
||||||
|
open: true,
|
||||||
|
pending: new Map(),
|
||||||
|
sequence: 0,
|
||||||
|
nested: (message.budget?.depth ?? 0) > 1,
|
||||||
|
};
|
||||||
// Descendants cannot wait behind ancestors that hold all nested capacity.
|
// Descendants cannot wait behind ancestors that hold all nested capacity.
|
||||||
// Reject excess fan-out/depth rather than deadlock an admitted call tree.
|
// Reject excess fan-out/depth rather than deadlock an admitted call tree.
|
||||||
if (!available(call) && (call.nested || queued.length >= limits.maxQueued)) {
|
if (!available(call) && (call.nested || queued.length >= limits.maxQueued)) {
|
||||||
@@ -494,7 +591,16 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
io.stdin.on("error", (error) => void fail(error));
|
io.stdin.on("error", (error) => void fail(error));
|
||||||
io.stdout.on("error", (error) => void fail(error));
|
io.stdout.on("error", (error) => void fail(error));
|
||||||
send({ kind: "ready", exports: config.exports });
|
send({ kind: "ready", exports: config.exports });
|
||||||
return { close: () => fail(fixedError("SERVICE_SHUTDOWN")) };
|
return {
|
||||||
|
close: () => fail(fixedError("SERVICE_SHUTDOWN")),
|
||||||
|
stats: () => ({
|
||||||
|
...metrics,
|
||||||
|
active: active.size,
|
||||||
|
queued: queued.length,
|
||||||
|
engines: engines.size,
|
||||||
|
idle: idleEngines.length,
|
||||||
|
}),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (process.argv[1] === new URL(import.meta.url).pathname) {
|
if (process.argv[1] === new URL(import.meta.url).pathname) {
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ async function fixture(source, options = {}) {
|
|||||||
send,
|
send,
|
||||||
start,
|
start,
|
||||||
usage,
|
usage,
|
||||||
|
stats: service.stats,
|
||||||
async close() {
|
async close() {
|
||||||
await service.close();
|
await service.close();
|
||||||
await rm(directory, { recursive: true, force: true });
|
await rm(directory, { recursive: true, force: true });
|
||||||
@@ -424,3 +425,149 @@ test("queue wait consumes the deadline and cancelled work never reaches its port
|
|||||||
await f.close();
|
await f.close();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("warm engines create fresh transitive module state, closures and built-ins", async () => {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "qx-transitive-"));
|
||||||
|
const helper = join(directory, "helper.mjs");
|
||||||
|
await writeFile(
|
||||||
|
helper,
|
||||||
|
"let value=0, retained; export function touch(channel){const old=retained; retained=channel; return {count:++value, captured:!!old};}",
|
||||||
|
);
|
||||||
|
const f = await fixture(
|
||||||
|
`import {touch} from 'helper.mjs'; let count=0;
|
||||||
|
export const registry={exports:['test'],async invoke(request,channel){
|
||||||
|
const before={object:Object.prototype.retained??null,array:Array.prototype.retained??null,
|
||||||
|
map:Map.prototype.retained??null, global:globalThis.retained??null};
|
||||||
|
Object.prototype.retained=request.input; Array.prototype.retained=request.input;
|
||||||
|
Map.prototype.retained=request.input; globalThis.retained=request.input;
|
||||||
|
return {count:++count, helper:touch(channel), before};
|
||||||
|
}};`,
|
||||||
|
{ sdk: [{ name: "helper.mjs", path: helper }] },
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
for (const value of ["private graph A", "private graph B", "private graph C"]) {
|
||||||
|
f.start(value);
|
||||||
|
assert.deepEqual((await f.receive()).output, {
|
||||||
|
count: 1,
|
||||||
|
helper: { count: 1, captured: false },
|
||||||
|
before: { object: null, array: null, map: null, global: null },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
assert.equal(f.stats().starts, 1, "Qualification must exercise actual engine reuse");
|
||||||
|
assert.equal(f.stats().reused, 2);
|
||||||
|
assert.equal(f.stats().idle, 1);
|
||||||
|
} finally {
|
||||||
|
await f.close();
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an expired frame token cannot borrow a later invocation's broker", async () => {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "qx-token-"));
|
||||||
|
const worker = join(directory, "token-test.mjs");
|
||||||
|
// Trusted test entrypoint deliberately replays the old token. Authored
|
||||||
|
// registries never receive CONFIG or BROKER directly.
|
||||||
|
await writeFile(
|
||||||
|
worker,
|
||||||
|
`export default {async fetch(request,env) {
|
||||||
|
const message=await request.json();
|
||||||
|
const output=message.input ? (await env.BROKER.fetch('http://frame/port/'+message.input,{
|
||||||
|
method:'POST',body:JSON.stringify({portId:'read',action:{kind:'state',primitive:'read'},input:null})})).status
|
||||||
|
: env.CONFIG.frameToken;
|
||||||
|
return Response.json({status:'returned',output});
|
||||||
|
}};`,
|
||||||
|
);
|
||||||
|
const f = await fixture(registry("return null;"), { worker });
|
||||||
|
try {
|
||||||
|
f.start();
|
||||||
|
const token = (await f.receive()).output;
|
||||||
|
assert.match(token, /^[a-f0-9-]{36}$/);
|
||||||
|
f.start(token);
|
||||||
|
assert.equal((await f.receive()).output, 409);
|
||||||
|
assert.equal(f.stats().starts, 1);
|
||||||
|
assert.equal(f.stats().reused, 1);
|
||||||
|
} finally {
|
||||||
|
await f.close();
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("idle engines evict without a graph poll and restart with fresh state", async () => {
|
||||||
|
const f = await fixture(registry("return request.input;"), { engineIdleMs: 30 });
|
||||||
|
try {
|
||||||
|
f.start("first");
|
||||||
|
assert.equal((await f.receive()).output, "first");
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||||
|
assert.equal(f.stats().engines, 0);
|
||||||
|
f.start("second");
|
||||||
|
assert.equal((await f.receive()).output, "second");
|
||||||
|
assert.equal(f.stats().starts, 2);
|
||||||
|
} finally {
|
||||||
|
await f.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const loop of [
|
||||||
|
"while(true) await Promise.resolve();",
|
||||||
|
"const memory=[]; for(let n=0;n<128;n++) memory.push(new Uint8Array(256*1024)); while(true) await Promise.resolve(memory);",
|
||||||
|
])
|
||||||
|
test("await/allocation pressure is fenced without delaying a neighboring engine: " + loop.slice(0, 28), async () => {
|
||||||
|
const f = await fixture(registry(`if(request.input==='spin'){${loop}} return request.input;`), {
|
||||||
|
deadlineMs: 1500,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const stuck = f.start("spin", { root: "stuck", depth: 1, deadlineMs: 350, calls: 10 });
|
||||||
|
const before = performance.now();
|
||||||
|
const healthy = f.start("healthy");
|
||||||
|
const replies = [await f.receive(), await f.receive()];
|
||||||
|
assert.equal(replies.find((r) => r.invocation === healthy.invocation).output, "healthy");
|
||||||
|
assert.equal(replies.find((r) => r.invocation === stuck.invocation).error.code, "RESTRICTED_DEADLINE");
|
||||||
|
assert.equal(replies[0].invocation, healthy.invocation, "Healthy work must finish before the stuck deadline");
|
||||||
|
assert.ok(performance.now() - before < 1200);
|
||||||
|
f.start("after fault");
|
||||||
|
assert.equal((await f.receive()).output, "after fault");
|
||||||
|
assert.equal(f.stats().starts, 2, "The healthy engine survives its neighbor's termination");
|
||||||
|
} finally {
|
||||||
|
await f.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("detached microtask loops cannot escape the invocation wall deadline", async () => {
|
||||||
|
const f = await fixture(
|
||||||
|
registry("void(async()=>{while(true)await Promise.resolve();})(); return 'nominal return';"),
|
||||||
|
{ deadlineMs: 200 },
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
const before = performance.now();
|
||||||
|
f.start();
|
||||||
|
assert.equal((await f.receive()).error.code, "RESTRICTED_DEADLINE");
|
||||||
|
assert.ok(performance.now() - before < 1000);
|
||||||
|
assert.equal(f.stats().engines, 0);
|
||||||
|
} finally {
|
||||||
|
await f.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("detached digest callbacks cannot use a completed frame or contaminate reuse", async () => {
|
||||||
|
const f = await fixture(
|
||||||
|
registry(`
|
||||||
|
void(async()=>{
|
||||||
|
for(let n=0;n<100;n++)await crypto.subtle.digest('SHA-256',new Uint8Array(32768));
|
||||||
|
await channel({portId:'late',action:{kind:'state',primitive:'read'},input:request.input});
|
||||||
|
})().catch(()=>{});
|
||||||
|
return request.input;
|
||||||
|
`),
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
f.start("private A");
|
||||||
|
assert.equal((await f.receive()).output, "private A");
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 75));
|
||||||
|
f.start("private B");
|
||||||
|
const reply = await f.receive();
|
||||||
|
assert.equal(reply.status, "returned", "No late graph port may appear after completion");
|
||||||
|
assert.equal(reply.output, "private B");
|
||||||
|
assert.equal(f.stats().starts, 1);
|
||||||
|
} finally {
|
||||||
|
await f.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
// Platform entrypoint, evaluated before loading the authored registry. A fresh
|
// Platform entrypoint, evaluated before loading the authored registry. A fresh
|
||||||
// engine process owns exactly one frame. BROKER grants only that frame's ports.
|
// Worker owns exactly one frame. The parent engine is reusable; BROKER's host
|
||||||
|
// token grants only this frame and expires before the next invocation.
|
||||||
|
const monotonic = performance.now.bind(performance);
|
||||||
const deny = () => {
|
const deny = () => {
|
||||||
throw Error("RESTRICTED_AMBIENT_ACCESS");
|
throw Error("RESTRICTED_AMBIENT_ACCESS");
|
||||||
};
|
};
|
||||||
@@ -48,7 +50,9 @@ export default {
|
|||||||
if (used) return new Response("FRAME_EXPIRED", { status: 409 });
|
if (used) return new Response("FRAME_EXPIRED", { status: 409 });
|
||||||
used = true;
|
used = true;
|
||||||
const input = await request.json();
|
const input = await request.json();
|
||||||
|
const loading = monotonic();
|
||||||
const source = await import("./package.mjs");
|
const source = await import("./package.mjs");
|
||||||
|
const initializationMs = monotonic() - loading;
|
||||||
const registry = source[env.CONFIG.registryExport];
|
const registry = source[env.CONFIG.registryExport];
|
||||||
if (
|
if (
|
||||||
!registry ||
|
!registry ||
|
||||||
@@ -59,7 +63,7 @@ export default {
|
|||||||
let open = true;
|
let open = true;
|
||||||
const channel = async (value) => {
|
const channel = async (value) => {
|
||||||
if (!open) throw Error("EXPIRED_CONTEXT");
|
if (!open) throw Error("EXPIRED_CONTEXT");
|
||||||
const response = await env.BROKER.fetch("http://frame/port", {
|
const response = await env.BROKER.fetch("http://frame/port/" + env.CONFIG.frameToken, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
body: JSON.stringify(value),
|
body: JSON.stringify(value),
|
||||||
});
|
});
|
||||||
@@ -71,6 +75,7 @@ export default {
|
|||||||
});
|
});
|
||||||
return result.output;
|
return result.output;
|
||||||
};
|
};
|
||||||
|
const running = monotonic();
|
||||||
try {
|
try {
|
||||||
const output = await registry.invoke(
|
const output = await registry.invoke(
|
||||||
{ exportId: input.exportId, workspace: input.workspace, receiver: input.receiver, input: input.input },
|
{ exportId: input.exportId, workspace: input.workspace, receiver: input.receiver, input: input.input },
|
||||||
@@ -78,7 +83,13 @@ export default {
|
|||||||
);
|
);
|
||||||
const body = JSON.stringify({ status: "returned", output });
|
const body = JSON.stringify({ status: "returned", output });
|
||||||
if (new TextEncoder().encode(body).length > env.CONFIG.maxBytes) throw Error("RESTRICTED_OUTPUT_LIMIT");
|
if (new TextEncoder().encode(body).length > env.CONFIG.maxBytes) throw Error("RESTRICTED_OUTPUT_LIMIT");
|
||||||
return new Response(body, { headers: { "content-type": "application/json" } });
|
return new Response(body, {
|
||||||
|
headers: {
|
||||||
|
"content-type": "application/json",
|
||||||
|
"x-quixos-initialization-ms": String(initializationMs),
|
||||||
|
"x-quixos-execution-ms": String(monotonic() - running),
|
||||||
|
},
|
||||||
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Only authored/codec rejection codes cross the engine boundary. Engine
|
// Only authored/codec rejection codes cross the engine boundary. Engine
|
||||||
// diagnostics stay private to the supervisor, never become protocol data.
|
// diagnostics stay private to the supervisor, never become protocol data.
|
||||||
|
|||||||
Reference in New Issue
Block a user