From 391f02f78a54aece11c06ff8330bd3955cbf808f Mon Sep 17 00:00:00 2001 From: "Timothy J. Aveni" Date: Wed, 7 Oct 2026 16:14:15 -0700 Subject: [PATCH] Pool host-approved restricted code with scoped and fair graph execution --- src/execution-world/candidate.ts | 8 ++- src/execution-world/files.ts | 44 +++++++++------- src/execution-world/metadata.ts | 87 +++++++++++++++++++++++++------- src/execution-world/placement.ts | 5 +- test/candidate-metadata.test.ts | 61 ++++++++++++++++++++-- 5 files changed, 160 insertions(+), 45 deletions(-) diff --git a/src/execution-world/candidate.ts b/src/execution-world/candidate.ts index 171a5fb..090cd57 100644 --- a/src/execution-world/candidate.ts +++ b/src/execution-world/candidate.ts @@ -11,8 +11,12 @@ import { } from "./metadata.js"; const rootPattern = /^\/nix\/store\/[a-z0-9]{32}-[^/\s]+$/; async function readJson(root: string, file: string, scope: CandidateMetadataScope): Promise { - const bytes = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024); - return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + const input = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024); + try { + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(input.bytes)); + } finally { + input.releaseInput(); + } } /** Load the checked Nix output, preserving its original compilation and provenance. * The selected toolchain validates generated code during the build. This loader diff --git a/src/execution-world/files.ts b/src/execution-world/files.ts index 4c59a09..d28166e 100644 --- a/src/execution-world/files.ts +++ b/src/execution-world/files.ts @@ -13,7 +13,7 @@ import { type CheckedWorld, } from "./artifacts.js"; import { type ExecutionPlan, ExecutionPlanError } from "./plan.js"; -import type { CandidateMetadataScope } from "./metadata.js"; +import { readCandidateMetadata, type CandidateMetadataScope } from "./metadata.js"; const maxFileBytes = 256 * 1024 * 1024; const metadataLimit = 16 * 1024 * 1024; @@ -22,10 +22,19 @@ const fail = (where: string, message: string): never => { }; /** Roots must be trusted immutable output directories. Refuse symlinks, including * interior directory links. Hash verification is still required before serving bytes. */ -export async function artifactFileReader( - roots: Readonly>, - metadata?: CandidateMetadataScope, -): Promise { +async function artifactFile(root: string, relative: string): Promise { + artifactPath(relative); + const components = relative.split("/"); + let resolved = root; + for (let index = 0; index < components.length; index++) { + resolved = path.join(resolved, components[index]!); + const stat = await lstat(resolved); + if (stat.isSymbolicLink() || (index < components.length - 1 ? !stat.isDirectory() : !stat.isFile())) + fail(relative, "Only real directories and regular files are allowed"); + } + return resolved; +} +export async function artifactFileReader(roots: Readonly>): Promise { const directories = new Map(); for (const [revision, root] of Object.entries(roots)) { if (!path.isAbsolute(root) || path.normalize(root) !== root || (await realpath(root)) !== root) @@ -37,21 +46,12 @@ export async function artifactFileReader( return async (revision, relative, limit) => { const root = directories.get(revision); if (!root) fail(revision, "Unknown package output"); - artifactPath(relative); if (!Number.isSafeInteger(limit) || limit < 0 || limit > maxFileBytes) fail(relative, "Invalid file byte limit"); - const components = relative.split("/"); - let resolved = root!; - for (let index = 0; index < components.length; index++) { - resolved = path.join(resolved, components[index]!); - const stat = await lstat(resolved); - if (stat.isSymbolicLink() || (index < components.length - 1 ? !stat.isDirectory() : !stat.isFile())) - fail(relative, "Only real directories and regular files are allowed"); - } + const resolved = await artifactFile(root!, relative); const handle = await open(resolved, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); try { const stat = await handle.stat(); if (!stat.isFile() || stat.size > limit) fail(relative, "File exceeds its declared bound"); - if (relative === "share/quixos/package-artifacts.json") metadata?.charge((stat.size + 1) * 32); // A fixed allocation and one extra byte detect growth without readFile's unbounded allocation. const buffer = Buffer.alloc(stat.size + 1); let offset = 0; @@ -79,14 +79,22 @@ export async function loadArtifactWorld( roots: Readonly>, metadata?: CandidateMetadataScope, ): Promise { - const read = await artifactFileReader(roots, metadata), + const read = await artifactFileReader(roots), manifests: PackageArtifacts[] = []; const expected = plan.definition.packages.map((p) => p.revision).sort(); if (JSON.stringify(Object.keys(roots).sort()) !== JSON.stringify(expected)) fail("world", "Output roots must exactly match the installed plan"); for (const revision of expected) { - const bytes = await read(revision, "share/quixos/package-artifacts.json", metadataLimit); - const manifest = parsePackageArtifacts(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))); + const file = "share/quixos/package-artifacts.json"; + const input = metadata + ? await readCandidateMetadata(await artifactFile(roots[revision]!, file), metadata, metadataLimit) + : { bytes: await read(revision, file, metadataLimit), releaseInput: () => {} }; + let manifest: PackageArtifacts; + try { + manifest = parsePackageArtifacts(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(input.bytes))); + } finally { + input.releaseInput(); + } if (manifest.packageRevisionId !== revision) fail(revision, "Output belongs to another package revision"); manifests.push(manifest); } diff --git a/src/execution-world/metadata.ts b/src/execution-world/metadata.ts index 847b993..a7c0664 100644 --- a/src/execution-world/metadata.ts +++ b/src/execution-world/metadata.ts @@ -24,28 +24,44 @@ export class CandidateMetadataBudget { this.scopes++; let bytes = 0, closed = false; + const charge = (amount: number) => { + if (closed) throw Error("CANDIDATE_METADATA_CLOSED"); + if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE"); + const owned = this.owners.get(owner) ?? 0; + if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes) + throw Error("CANDIDATE_METADATA_LIMIT"); + bytes += amount; + this.bytes += amount; + this.owners.set(owner, owned + amount); + }; + const subtract = (amount: number) => { + bytes -= amount; + this.bytes -= amount; + const remaining = (this.owners.get(owner) ?? 0) - amount; + if (remaining) this.owners.set(owner, remaining); + else this.owners.delete(owner); + }; return { get bytes() { return bytes; }, - charge: (amount: number) => { - if (closed) throw Error("CANDIDATE_METADATA_CLOSED"); - if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE"); - const owned = this.owners.get(owner) ?? 0; - if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes) - throw Error("CANDIDATE_METADATA_LIMIT"); - bytes += amount; - this.bytes += amount; - this.owners.set(owner, owned + amount); + charge, + /** Opaque, idempotent transient reservation; callers cannot subtract an + * arbitrary amount or release another candidate's charges. */ + reserve: (amount: number) => { + charge(amount); + let released = false; + return () => { + if (closed || released) return; + released = true; + subtract(amount); + }; }, release: () => { if (closed) return; closed = true; - this.bytes -= bytes; + subtract(bytes); this.scopes--; - const remaining = (this.owners.get(owner) ?? 0) - bytes; - if (remaining) this.owners.set(owner, remaining); - else this.owners.delete(owner); }, }; } @@ -53,21 +69,50 @@ export class CandidateMetadataBudget { export type CandidateMetadataScope = ReturnType; export const candidateMetadataBudget = new CandidateMetadataBudget(); +// Estimate JSON allocation without constructing its tree. Object/array headers, +// value slots and strings get explicit weights, with a multiplier for checked +// indexes/derived representations. Indentation must not multiply tree charges. +function jsonAllocation(bytes: Uint8Array): number { + let estimate = 0; + for (let i = 0; i < bytes.length; i++) { + const c = bytes[i]!; + if (c === 123) estimate += 64; + else if (c === 91) estimate += 32; + else if (c === 34) { + const start = ++i; + while (i < bytes.length && bytes[i] !== 34) { + if (bytes[i] === 92) i++; + i++; + } + estimate += 32 + 2 * (i - start); + } else if (c === 45 || (c >= 48 && c <= 57) || c === 116 || c === 102 || c === 110) { + estimate += 16; + while (i + 1 < bytes.length) { + const next = bytes[i + 1]!; + if (next === 9 || next === 10 || next === 13 || next === 32 || next === 44 || next === 93 || next === 125) + break; + i++; + } + } + } + return estimate * 2; +} + /** Immutable regular files only. Fixed allocation plus an extra byte catches * growth; readFile() would allocate before the admission check can intervene. */ export async function readCandidateMetadata( path: string, scope: CandidateMetadataScope, limit: number, -): Promise { +): Promise<{ bytes: Uint8Array; releaseInput: () => void }> { const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let releaseInput: (() => void) | undefined; try { const stat = await handle.stat(); if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size > limit) throw Error("CANDIDATE_METADATA_FILE_LIMIT"); - // Conservative attribution includes strings, small JSON nodes, duplicate - // checked representations, digest serialization and installed indexes. - scope.charge((stat.size + 1) * 32); + // Reserve the input buffer and UTF-16 decoding before allocating either. + releaseInput = scope.reserve((stat.size + 1) * 3); const bytes = Buffer.alloc(stat.size + 1); let offset = 0; while (offset < bytes.length) { @@ -76,7 +121,13 @@ export async function readCandidateMetadata( offset += read.bytesRead; } if (offset !== stat.size) throw Error("CANDIDATE_METADATA_CHANGED"); - return bytes.subarray(0, offset); + const content = bytes.subarray(0, offset); + // No JSON.parse/decoded tree exists until its shape-based charge succeeds. + scope.charge(jsonAllocation(content)); + return { bytes: content, releaseInput }; + } catch (error) { + releaseInput?.(); + throw error; } finally { await handle.close(); } diff --git a/src/execution-world/placement.ts b/src/execution-world/placement.ts index 468e191..1b8fe01 100644 --- a/src/execution-world/placement.ts +++ b/src/execution-world/placement.ts @@ -137,8 +137,9 @@ export function inspectJavaScriptSurface(source: string, checkedSDKImports: read } /** Derived metadata only. Browser/server portability and available targets say - * nothing about sandbox eligibility. No existing package artifact has a - * qualified restricted runtime, so it conservatively requires its VM. */ + * nothing about sandbox eligibility. Build reports have no platform approval + * catalog, so they conservatively require VM placement. A host can override + * placement only for independently reviewed, exact restricted-code approvals. */ export async function reportExecutionPlacement( plan: ExecutionPlan, installation: ExecutionInstallation, diff --git a/test/candidate-metadata.test.ts b/test/candidate-metadata.test.ts index 103dff3..b8871f1 100644 --- a/test/candidate-metadata.test.ts +++ b/test/candidate-metadata.test.ts @@ -28,23 +28,74 @@ test("concurrent candidate reservations enforce aggregate and owner limits and r test("metadata files reserve parse/index space before allocation and refuse oversized or linked inputs", async () => { const root = await mkdtemp(join(tmpdir(), "qx-metadata-budget-")); - const budget = new CandidateMetadataBudget({ bytes: 512, ownerBytes: 512, candidates: 2 }); + const budget = new CandidateMetadataBudget({ bytes: 450, ownerBytes: 450, candidates: 2 }); const scope = budget.acquire("workspace"); try { const file = join(root, "metadata.json"); await writeFile(file, '{"ok":true}'); - assert.equal(new TextDecoder().decode(await readCandidateMetadata(file, scope, 100)), '{"ok":true}'); - assert.equal(scope.bytes, 12 * 32); + const input = await readCandidateMetadata(file, scope, 100); + assert.equal(new TextDecoder().decode(input.bytes), '{"ok":true}'); + const withInput = scope.bytes; + input.releaseInput(); + input.releaseInput(); + const first = scope.bytes; + assert.equal(withInput - first, (Buffer.byteLength('{"ok":true}') + 1) * 3); + assert.ok(first > 12 * 3, "Decoded tree and derived metadata are reserved before parsing"); await assert.rejects(readCandidateMetadata(file, scope, 100), /CANDIDATE_METADATA_LIMIT/); - assert.equal(scope.bytes, 12 * 32); + assert.equal(scope.bytes, first, "Rejected tree allocation releases its transient input buffer"); await assert.rejects(readCandidateMetadata(file, scope, 5), /CANDIDATE_METADATA_FILE_LIMIT/); await symlink(file, join(root, "linked")); await assert.rejects(readCandidateMetadata(join(root, "linked"), scope, 100), { code: "ELOOP" }); await assert.rejects(readCandidateMetadata(root, scope, 100), /CANDIDATE_METADATA_FILE_LIMIT/); - assert.equal(scope.bytes, 12 * 32); + assert.equal(scope.bytes, first); } finally { scope.release(); await rm(root, { recursive: true, force: true }); } assert.equal(budget.snapshot().bytes, 0); }); + +test("JSON structure accounting is independent of indentation and covers strings, values and dense nodes", async () => { + const root = await mkdtemp(join(tmpdir(), "qx-metadata-shape-")); + const budget = new CandidateMetadataBudget(); + try { + for (const value of [{}, [null, true, false, 1, -3.5e8], { 'a"b': '雪\\"', nodes: [{}, {}, {}] }]) { + const measured = []; + for (const spaces of [0, 4]) { + const file = join(root, "metadata.json"), + text = JSON.stringify(value, null, spaces); + await writeFile(file, text); + const scope = budget.acquire("test"); + try { + const input = await readCandidateMetadata(file, scope, 10000); + assert.deepEqual(JSON.parse(new TextDecoder().decode(input.bytes)), value); + input.releaseInput(); + measured.push(scope.bytes); + } finally { + scope.release(); + } + } + assert.equal(measured[0], measured[1], "Whitespace changes input storage, not the allocated JSON shape"); + assert.ok(measured[0] >= 128); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + assert.equal(budget.snapshot().bytes, 0); +}); + +test("transient leases release exactly once, including after their parent closes", () => { + const budget = new CandidateMetadataBudget({ bytes: 1000, ownerBytes: 600, candidates: 3 }); + const scope = budget.acquire("a"); + scope.charge(200); + const release = scope.reserve(300); + assert.throws(() => scope.reserve(101), /CANDIDATE_METADATA_LIMIT/); + release(); + release(); + assert.equal(scope.bytes, 200); + const afterClose = scope.reserve(100); + scope.release(); + afterClose(); + assert.equal(scope.bytes, 0); + assert.equal(budget.snapshot().bytes, 0); +});