From c8c56d8ea77d0abe2baf0d08eb2537b32e73f6d6 Mon Sep 17 00:00:00 2001 From: "Timothy J. Aveni" Date: Wed, 7 Oct 2026 14:44:59 -0700 Subject: [PATCH 1/3] Bound checked candidate metadata and restricted invocation resources --- src/execution-world/candidate.ts | 231 +++++++++++++++------------- src/execution-world/files.ts | 10 +- src/execution-world/index.ts | 1 + src/execution-world/installation.ts | 4 +- src/execution-world/metadata.ts | 83 ++++++++++ src/execution-world/ports.ts | 2 + test/candidate-metadata.test.ts | 50 ++++++ 7 files changed, 271 insertions(+), 110 deletions(-) create mode 100644 src/execution-world/metadata.ts create mode 100644 test/candidate-metadata.test.ts diff --git a/src/execution-world/candidate.ts b/src/execution-world/candidate.ts index 1be6d37..171a5fb 100644 --- a/src/execution-world/candidate.ts +++ b/src/execution-world/candidate.ts @@ -1,19 +1,18 @@ import { graphExecutionRepresentation } from "./graph-execution.js"; -import { open, lstat } from "node:fs/promises"; -import { constants } from "node:fs"; +import { lstat } from "node:fs/promises"; import { join } from "node:path"; import { hexDigest, ExecutionPlanError, type ExecutionPlan } from "./plan.js"; import { loadExecutionInstallation, type PackageInstallationSelection } from "./installation.js"; +import { + candidateMetadataBudget, + readCandidateMetadata, + type CandidateMetadataBudget, + type CandidateMetadataScope, +} from "./metadata.js"; const rootPattern = /^\/nix\/store\/[a-z0-9]{32}-[^/\s]+$/; -async function readJson(root: string, file: string): Promise { - const handle = await open(join(root, file), constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const metadata = await handle.stat(); - if (!metadata.isFile() || metadata.size > 32 * 1024 * 1024) throw Error(`Invalid checked candidate file: ${file}`); - return JSON.parse(await handle.readFile("utf8")); - } finally { - await handle.close(); - } +async function readJson(root: string, file: string, scope: CandidateMetadataScope): Promise { + const bytes = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024); + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); } /** Load the checked Nix output, preserving its original compilation and provenance. * The selected toolchain validates generated code during the build. This loader @@ -22,113 +21,131 @@ async function readJson(root: string, file: string): Promise { export async function loadCheckedInstallationCandidate( root: string, selections: Readonly> = {}, + resources: { budget?: CandidateMetadataBudget; owner?: string } = {}, ) { if (!rootPattern.test(root) || !(await lstat(root)).isDirectory()) throw Error("Candidate must be an immutable Nix output directory"); - const [candidate, plan, artifacts, source, entryRequest, kernel, queries, graph] = await Promise.all([ - readJson(root, "candidate.json"), - readJson(root, "execution-plan.json"), - readJson(root, "execution-artifacts.json"), - readJson(root, "source.json"), - readJson(root, "entry.json"), - readJson(root, "kernel.json"), - readJson(root, "queries.json"), - readJson(root, "graph-execution.json"), - ]); - const { executionContracts: _legacyContracts, ...workspace } = candidate; - if ( - plan?.definition?.codec !== "camino-execution-plan-v1" || - hexDigest(plan.definition) !== plan.digest || - hexDigest(workspace) !== hexDigest(plan.definition.workspace) - ) - throw new ExecutionPlanError( - "CANDIDATE_PLAN_MISMATCH", - root, - "Stored plan must match its digest and exact workspace source contract", - ); - if ( - source.kind !== "workspace" || - source.commit !== plan.definition.workspace.sourceRootCommit || - typeof source.repository !== "string" - ) - throw new ExecutionPlanError("CANDIDATE_SOURCE_MISMATCH", root, "Expected exact workspace source provenance"); - const repository = new URL(source.repository); - if ( - !["https:", "ssh:"].includes(repository.protocol) || - repository.username || - repository.password || - repository.hash || - repository.search - ) - throw new ExecutionPlanError("CANDIDATE_SOURCE_INVALID", root, "Expected a source URL without credentials"); - if ( - artifacts.codec !== "camino-checked-execution-v2" || - typeof artifacts.generator !== "string" || - !/^[0-9a-f]{64}$/.test(artifacts.generator) || - !artifacts.roots || - typeof artifacts.roots !== "object" || - Array.isArray(artifacts.roots) - ) - throw new ExecutionPlanError("CANDIDATE_ARTIFACTS_INVALID", root, "Expected checked package roots"); - for (const entry of Object.values(artifacts.roots)) - if (typeof entry !== "string" || !rootPattern.test(entry)) + const scope = (resources.budget ?? candidateMetadataBudget).acquire(resources.owner ?? root); + try { + // Sequential reads avoid partially rejected Promise.all() work allocating + // after the failed candidate's reservation has been released. + const values = []; + for (const file of [ + "candidate.json", + "execution-plan.json", + "execution-artifacts.json", + "source.json", + "entry.json", + "kernel.json", + "queries.json", + "graph-execution.json", + ]) + values.push(await readJson(root, file, scope)); + const [candidate, plan, artifacts, source, entryRequest, kernel, queries, graph] = values; + const { executionContracts: _legacyContracts, ...workspace } = candidate; + if ( + plan?.definition?.codec !== "camino-execution-plan-v1" || + hexDigest(plan.definition) !== plan.digest || + hexDigest(workspace) !== hexDigest(plan.definition.workspace) + ) throw new ExecutionPlanError( - "CANDIDATE_ARTIFACTS_INVALID", + "CANDIDATE_PLAN_MISMATCH", root, - "Package artifacts must be immutable Nix roots", + "Stored plan must match its digest and exact workspace source contract", ); - const installation = await loadExecutionInstallation(plan, artifacts.roots, selections); - const entry = compileWorkspaceEntry(plan, entryRequest); - for (const [name, program, codec] of [ - ["kernel", kernel, "camino-kernel-program-v1"], - ["queries", queries, "camino-query-program-v1"], - ] as const) - if (program?.codec !== codec || program.plan !== kernel.plan || hexDigest(program) !== artifacts.programs?.[name]) + if ( + source.kind !== "workspace" || + source.commit !== plan.definition.workspace.sourceRootCommit || + typeof source.repository !== "string" + ) + throw new ExecutionPlanError("CANDIDATE_SOURCE_MISMATCH", root, "Expected exact workspace source provenance"); + const repository = new URL(source.repository); + if ( + !["https:", "ssh:"].includes(repository.protocol) || + repository.username || + repository.password || + repository.hash || + repository.search + ) + throw new ExecutionPlanError("CANDIDATE_SOURCE_INVALID", root, "Expected a source URL without credentials"); + if ( + artifacts.codec !== "camino-checked-execution-v2" || + typeof artifacts.generator !== "string" || + !/^[0-9a-f]{64}$/.test(artifacts.generator) || + !artifacts.roots || + typeof artifacts.roots !== "object" || + Array.isArray(artifacts.roots) + ) + throw new ExecutionPlanError("CANDIDATE_ARTIFACTS_INVALID", root, "Expected checked package roots"); + for (const entry of Object.values(artifacts.roots)) + if (typeof entry !== "string" || !rootPattern.test(entry)) + throw new ExecutionPlanError( + "CANDIDATE_ARTIFACTS_INVALID", + root, + "Package artifacts must be immutable Nix roots", + ); + const installation = await loadExecutionInstallation(plan, artifacts.roots, selections, scope); + const entry = compileWorkspaceEntry(plan, entryRequest); + for (const [name, program, codec] of [ + ["kernel", kernel, "camino-kernel-program-v1"], + ["queries", queries, "camino-query-program-v1"], + ] as const) + if (program?.codec !== codec || program.plan !== kernel.plan || hexDigest(program) !== artifacts.programs?.[name]) + throw new ExecutionPlanError( + "CANDIDATE_PROGRAM_MISMATCH", + name, + "Stored interpreter input must match the checked build and plan", + ); + const expectedGraph = graphExecutionRepresentation(plan, installation, kernel, queries); + if (hexDigest(graph) !== hexDigest(expectedGraph) || artifacts.execution !== hexDigest(graph)) throw new ExecutionPlanError( - "CANDIDATE_PROGRAM_MISMATCH", - name, - "Stored interpreter input must match the checked build and plan", + "CANDIDATE_EXECUTION_MISMATCH", + root, + "Stored graph execution differs from checked inputs", ); - const expectedGraph = graphExecutionRepresentation(plan, installation, kernel, queries); - if (hexDigest(graph) !== hexDigest(expectedGraph) || artifacts.execution !== hexDigest(graph)) - throw new ExecutionPlanError( - "CANDIDATE_EXECUTION_MISMATCH", + const provenance = { + workspaceRevision: workspace.id as string, + source: { repository: source.repository as string, commit: source.commit as string }, + generator: artifacts.generator as string, + packages: (plan as ExecutionPlan).definition.packages.map((pkg) => ({ + revision: pkg.revision, + source: pkg.source, + })), + }; + const reference = { root, - "Stored graph execution differs from checked inputs", - ); - const provenance = { - workspaceRevision: workspace.id as string, - source: { repository: source.repository as string, commit: source.commit as string }, - generator: artifacts.generator as string, - packages: (plan as ExecutionPlan).definition.packages.map((pkg) => ({ - revision: pkg.revision, - source: pkg.source, - })), - }; - const reference = { - root, - digest: hexDigest({ - plan: plan.digest, - world: installation.world, + digest: hexDigest({ + plan: plan.digest, + world: installation.world, + provenance, + roots: artifacts.roots, + entry, + programs: artifacts.programs, + }), + }; + return { + reference, + workspace: plan.definition.workspace.workspaceId, + world: hexDigest(graph), + plan: kernel.plan as string, + graphExecution: graph, provenance, - roots: artifacts.roots, entry, - programs: artifacts.programs, - }), - }; - return { - reference, - workspace: plan.definition.workspace.workspaceId, - world: hexDigest(graph), - plan: kernel.plan as string, - graphExecution: graph, - provenance, - entry, - kernel, - queries, - installation, - executionPlan: plan as ExecutionPlan, - }; + kernel, + queries, + installation, + executionPlan: plan as ExecutionPlan, + metadata: { + get bytes() { + return scope.bytes; + }, + release: scope.release, + }, + }; + } catch (error) { + scope.release(); + throw error; + } } export function compileWorkspaceEntry(plan: ExecutionPlan, entryRequest: unknown): { operation: string } | null { diff --git a/src/execution-world/files.ts b/src/execution-world/files.ts index f5b627a..4c59a09 100644 --- a/src/execution-world/files.ts +++ b/src/execution-world/files.ts @@ -13,6 +13,7 @@ import { type CheckedWorld, } from "./artifacts.js"; import { type ExecutionPlan, ExecutionPlanError } from "./plan.js"; +import type { CandidateMetadataScope } from "./metadata.js"; const maxFileBytes = 256 * 1024 * 1024; const metadataLimit = 16 * 1024 * 1024; @@ -21,7 +22,10 @@ const fail = (where: string, message: string): never => { }; /** Roots must be trusted immutable output directories. Refuse symlinks, including * interior directory links. Hash verification is still required before serving bytes. */ -export async function artifactFileReader(roots: Readonly>): Promise { +export async function artifactFileReader( + roots: Readonly>, + metadata?: CandidateMetadataScope, +): Promise { const directories = new Map(); for (const [revision, root] of Object.entries(roots)) { if (!path.isAbsolute(root) || path.normalize(root) !== root || (await realpath(root)) !== root) @@ -47,6 +51,7 @@ export async function artifactFileReader(roots: Readonly> try { const stat = await handle.stat(); if (!stat.isFile() || stat.size > limit) fail(relative, "File exceeds its declared bound"); + if (relative === "share/quixos/package-artifacts.json") metadata?.charge((stat.size + 1) * 32); // A fixed allocation and one extra byte detect growth without readFile's unbounded allocation. const buffer = Buffer.alloc(stat.size + 1); let offset = 0; @@ -72,8 +77,9 @@ export interface LoadedArtifactWorld { export async function loadArtifactWorld( plan: ExecutionPlan, roots: Readonly>, + metadata?: CandidateMetadataScope, ): Promise { - const read = await artifactFileReader(roots), + const read = await artifactFileReader(roots, metadata), manifests: PackageArtifacts[] = []; const expected = plan.definition.packages.map((p) => p.revision).sort(); if (JSON.stringify(Object.keys(roots).sort()) !== JSON.stringify(expected)) diff --git a/src/execution-world/index.ts b/src/execution-world/index.ts index a8e0b00..f605a24 100644 --- a/src/execution-world/index.ts +++ b/src/execution-world/index.ts @@ -17,6 +17,7 @@ export * from "./queries.js"; export * from "./query-document.js"; export { generatePortableClientBindings } from "../bindings/portable-client.js"; export * from "./candidate.js"; +export * from "./metadata.js"; export { reviewSemanticMajors } from "../capability-model/evolution.js"; diff --git a/src/execution-world/installation.ts b/src/execution-world/installation.ts index 61be4dc..3c9d26d 100644 --- a/src/execution-world/installation.ts +++ b/src/execution-world/installation.ts @@ -11,6 +11,7 @@ import { type PublicQueryContract, } from "./client-contracts.js"; import { resultInterfaces } from "./projections.js"; +import type { CandidateMetadataScope } from "./metadata.js"; /** Activation is assigned by graph authority, never by a package artifact. */ export interface ActiveExecutionIdentity { @@ -159,8 +160,9 @@ export async function loadExecutionInstallation( plan: ExecutionPlan, roots: Readonly>, selections: Readonly> = {}, + metadata?: CandidateMetadataScope, ): Promise { - return assembleExecutionInstallation(plan, roots, await loadArtifactWorld(plan, roots), selections); + return assembleExecutionInstallation(plan, roots, await loadArtifactWorld(plan, roots, metadata), selections); } function assembleExecutionInstallation( plan: ExecutionPlan, diff --git a/src/execution-world/metadata.ts b/src/execution-world/metadata.ts new file mode 100644 index 0000000..847b993 --- /dev/null +++ b/src/execution-world/metadata.ts @@ -0,0 +1,83 @@ +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; + +/** Admission estimates, not a hard V8 heap limit. A process/cgroup ceiling is + * still required. Charge input, parse trees and derived indexes before reading + * or parsing. Each live candidate retains its reservation until explicit close. */ +export class CandidateMetadataBudget { + readonly limits: Readonly<{ bytes: number; ownerBytes: number; candidates: number }>; + private bytes = 0; + private scopes = 0; + private readonly owners = new Map(); + constructor(limits = { bytes: 512 * 1024 * 1024, ownerBytes: 256 * 1024 * 1024, candidates: 128 }) { + for (const value of Object.values(limits)) + if (!Number.isSafeInteger(value) || value < 1) throw Error("INVALID_CANDIDATE_METADATA_BUDGET"); + if (limits.ownerBytes > limits.bytes) throw Error("INVALID_CANDIDATE_METADATA_BUDGET"); + this.limits = Object.freeze({ ...limits }); + } + snapshot() { + return { bytes: this.bytes, candidates: this.scopes, owners: this.owners.size, limits: this.limits }; + } + acquire(owner: string) { + if (!owner || owner.length > 4096) throw Error("INVALID_CANDIDATE_METADATA_OWNER"); + if (this.scopes >= this.limits.candidates) throw Error("CANDIDATE_METADATA_LIMIT"); + this.scopes++; + let bytes = 0, + closed = false; + return { + get bytes() { + return bytes; + }, + charge: (amount: number) => { + if (closed) throw Error("CANDIDATE_METADATA_CLOSED"); + if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE"); + const owned = this.owners.get(owner) ?? 0; + if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes) + throw Error("CANDIDATE_METADATA_LIMIT"); + bytes += amount; + this.bytes += amount; + this.owners.set(owner, owned + amount); + }, + release: () => { + if (closed) return; + closed = true; + this.bytes -= bytes; + this.scopes--; + const remaining = (this.owners.get(owner) ?? 0) - bytes; + if (remaining) this.owners.set(owner, remaining); + else this.owners.delete(owner); + }, + }; + } +} +export type CandidateMetadataScope = ReturnType; +export const candidateMetadataBudget = new CandidateMetadataBudget(); + +/** Immutable regular files only. Fixed allocation plus an extra byte catches + * growth; readFile() would allocate before the admission check can intervene. */ +export async function readCandidateMetadata( + path: string, + scope: CandidateMetadataScope, + limit: number, +): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size > limit) + throw Error("CANDIDATE_METADATA_FILE_LIMIT"); + // Conservative attribution includes strings, small JSON nodes, duplicate + // checked representations, digest serialization and installed indexes. + scope.charge((stat.size + 1) * 32); + const bytes = Buffer.alloc(stat.size + 1); + let offset = 0; + while (offset < bytes.length) { + const read = await handle.read(bytes, offset, bytes.length - offset, offset); + if (!read.bytesRead) break; + offset += read.bytesRead; + } + if (offset !== stat.size) throw Error("CANDIDATE_METADATA_CHANGED"); + return bytes.subarray(0, offset); + } finally { + await handle.close(); + } +} diff --git a/src/execution-world/ports.ts b/src/execution-world/ports.ts index d26e5a3..4a709bc 100644 --- a/src/execution-world/ports.ts +++ b/src/execution-world/ports.ts @@ -29,6 +29,8 @@ export interface PackageInvocation { /** Capability arguments/receiver use the versioned managed-reference codec. */ input: unknown; receiver: unknown | null; + /** Host-owned remaining root resources, separate from authored input. */ + budget?: { root: string; depth: number; calls: number; deadlineMs: number }; } export type InvocationReply = | { abi: typeof transactionPortAbi; invocation: string; status: "returned"; output: unknown } diff --git a/test/candidate-metadata.test.ts b/test/candidate-metadata.test.ts new file mode 100644 index 0000000..103dff3 --- /dev/null +++ b/test/candidate-metadata.test.ts @@ -0,0 +1,50 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile, symlink, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { CandidateMetadataBudget, readCandidateMetadata } from "../src/execution-world/metadata.js"; + +test("concurrent candidate reservations enforce aggregate and owner limits and release exactly once", () => { + const budget = new CandidateMetadataBudget({ bytes: 1000, ownerBytes: 600, candidates: 3 }); + const first = budget.acquire("a"), + second = budget.acquire("a"), + third = budget.acquire("b"); + assert.throws(() => budget.acquire("c"), /CANDIDATE_METADATA_LIMIT/); + first.charge(400); + second.charge(200); + third.charge(400); + assert.throws(() => second.charge(1), /CANDIDATE_METADATA_LIMIT/); + assert.throws(() => third.charge(1), /CANDIDATE_METADATA_LIMIT/); + assert.equal(budget.snapshot().bytes, 1000); + second.release(); + second.release(); + assert.equal(budget.snapshot().bytes, 800); + assert.throws(() => second.charge(1), /CANDIDATE_METADATA_CLOSED/); + first.release(); + third.release(); + assert.deepEqual(budget.snapshot(), { bytes: 0, candidates: 0, owners: 0, limits: budget.limits }); +}); + +test("metadata files reserve parse/index space before allocation and refuse oversized or linked inputs", async () => { + const root = await mkdtemp(join(tmpdir(), "qx-metadata-budget-")); + const budget = new CandidateMetadataBudget({ bytes: 512, ownerBytes: 512, candidates: 2 }); + const scope = budget.acquire("workspace"); + try { + const file = join(root, "metadata.json"); + await writeFile(file, '{"ok":true}'); + assert.equal(new TextDecoder().decode(await readCandidateMetadata(file, scope, 100)), '{"ok":true}'); + assert.equal(scope.bytes, 12 * 32); + await assert.rejects(readCandidateMetadata(file, scope, 100), /CANDIDATE_METADATA_LIMIT/); + assert.equal(scope.bytes, 12 * 32); + await assert.rejects(readCandidateMetadata(file, scope, 5), /CANDIDATE_METADATA_FILE_LIMIT/); + await symlink(file, join(root, "linked")); + await assert.rejects(readCandidateMetadata(join(root, "linked"), scope, 100), { code: "ELOOP" }); + await assert.rejects(readCandidateMetadata(root, scope, 100), /CANDIDATE_METADATA_FILE_LIMIT/); + assert.equal(scope.bytes, 12 * 32); + } finally { + scope.release(); + await rm(root, { recursive: true, force: true }); + } + assert.equal(budget.snapshot().bytes, 0); +}); From 391f02f78a54aece11c06ff8330bd3955cbf808f Mon Sep 17 00:00:00 2001 From: "Timothy J. Aveni" Date: Wed, 7 Oct 2026 16:14:15 -0700 Subject: [PATCH 2/3] Pool host-approved restricted code with scoped and fair graph execution --- src/execution-world/candidate.ts | 8 ++- src/execution-world/files.ts | 44 +++++++++------- src/execution-world/metadata.ts | 87 +++++++++++++++++++++++++------- src/execution-world/placement.ts | 5 +- test/candidate-metadata.test.ts | 61 ++++++++++++++++++++-- 5 files changed, 160 insertions(+), 45 deletions(-) diff --git a/src/execution-world/candidate.ts b/src/execution-world/candidate.ts index 171a5fb..090cd57 100644 --- a/src/execution-world/candidate.ts +++ b/src/execution-world/candidate.ts @@ -11,8 +11,12 @@ import { } from "./metadata.js"; const rootPattern = /^\/nix\/store\/[a-z0-9]{32}-[^/\s]+$/; async function readJson(root: string, file: string, scope: CandidateMetadataScope): Promise { - const bytes = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024); - return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + const input = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024); + try { + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(input.bytes)); + } finally { + input.releaseInput(); + } } /** Load the checked Nix output, preserving its original compilation and provenance. * The selected toolchain validates generated code during the build. This loader diff --git a/src/execution-world/files.ts b/src/execution-world/files.ts index 4c59a09..d28166e 100644 --- a/src/execution-world/files.ts +++ b/src/execution-world/files.ts @@ -13,7 +13,7 @@ import { type CheckedWorld, } from "./artifacts.js"; import { type ExecutionPlan, ExecutionPlanError } from "./plan.js"; -import type { CandidateMetadataScope } from "./metadata.js"; +import { readCandidateMetadata, type CandidateMetadataScope } from "./metadata.js"; const maxFileBytes = 256 * 1024 * 1024; const metadataLimit = 16 * 1024 * 1024; @@ -22,10 +22,19 @@ const fail = (where: string, message: string): never => { }; /** Roots must be trusted immutable output directories. Refuse symlinks, including * interior directory links. Hash verification is still required before serving bytes. */ -export async function artifactFileReader( - roots: Readonly>, - metadata?: CandidateMetadataScope, -): Promise { +async function artifactFile(root: string, relative: string): Promise { + artifactPath(relative); + const components = relative.split("/"); + let resolved = root; + for (let index = 0; index < components.length; index++) { + resolved = path.join(resolved, components[index]!); + const stat = await lstat(resolved); + if (stat.isSymbolicLink() || (index < components.length - 1 ? !stat.isDirectory() : !stat.isFile())) + fail(relative, "Only real directories and regular files are allowed"); + } + return resolved; +} +export async function artifactFileReader(roots: Readonly>): Promise { const directories = new Map(); for (const [revision, root] of Object.entries(roots)) { if (!path.isAbsolute(root) || path.normalize(root) !== root || (await realpath(root)) !== root) @@ -37,21 +46,12 @@ export async function artifactFileReader( return async (revision, relative, limit) => { const root = directories.get(revision); if (!root) fail(revision, "Unknown package output"); - artifactPath(relative); if (!Number.isSafeInteger(limit) || limit < 0 || limit > maxFileBytes) fail(relative, "Invalid file byte limit"); - const components = relative.split("/"); - let resolved = root!; - for (let index = 0; index < components.length; index++) { - resolved = path.join(resolved, components[index]!); - const stat = await lstat(resolved); - if (stat.isSymbolicLink() || (index < components.length - 1 ? !stat.isDirectory() : !stat.isFile())) - fail(relative, "Only real directories and regular files are allowed"); - } + const resolved = await artifactFile(root!, relative); const handle = await open(resolved, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); try { const stat = await handle.stat(); if (!stat.isFile() || stat.size > limit) fail(relative, "File exceeds its declared bound"); - if (relative === "share/quixos/package-artifacts.json") metadata?.charge((stat.size + 1) * 32); // A fixed allocation and one extra byte detect growth without readFile's unbounded allocation. const buffer = Buffer.alloc(stat.size + 1); let offset = 0; @@ -79,14 +79,22 @@ export async function loadArtifactWorld( roots: Readonly>, metadata?: CandidateMetadataScope, ): Promise { - const read = await artifactFileReader(roots, metadata), + const read = await artifactFileReader(roots), manifests: PackageArtifacts[] = []; const expected = plan.definition.packages.map((p) => p.revision).sort(); if (JSON.stringify(Object.keys(roots).sort()) !== JSON.stringify(expected)) fail("world", "Output roots must exactly match the installed plan"); for (const revision of expected) { - const bytes = await read(revision, "share/quixos/package-artifacts.json", metadataLimit); - const manifest = parsePackageArtifacts(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))); + const file = "share/quixos/package-artifacts.json"; + const input = metadata + ? await readCandidateMetadata(await artifactFile(roots[revision]!, file), metadata, metadataLimit) + : { bytes: await read(revision, file, metadataLimit), releaseInput: () => {} }; + let manifest: PackageArtifacts; + try { + manifest = parsePackageArtifacts(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(input.bytes))); + } finally { + input.releaseInput(); + } if (manifest.packageRevisionId !== revision) fail(revision, "Output belongs to another package revision"); manifests.push(manifest); } diff --git a/src/execution-world/metadata.ts b/src/execution-world/metadata.ts index 847b993..a7c0664 100644 --- a/src/execution-world/metadata.ts +++ b/src/execution-world/metadata.ts @@ -24,28 +24,44 @@ export class CandidateMetadataBudget { this.scopes++; let bytes = 0, closed = false; + const charge = (amount: number) => { + if (closed) throw Error("CANDIDATE_METADATA_CLOSED"); + if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE"); + const owned = this.owners.get(owner) ?? 0; + if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes) + throw Error("CANDIDATE_METADATA_LIMIT"); + bytes += amount; + this.bytes += amount; + this.owners.set(owner, owned + amount); + }; + const subtract = (amount: number) => { + bytes -= amount; + this.bytes -= amount; + const remaining = (this.owners.get(owner) ?? 0) - amount; + if (remaining) this.owners.set(owner, remaining); + else this.owners.delete(owner); + }; return { get bytes() { return bytes; }, - charge: (amount: number) => { - if (closed) throw Error("CANDIDATE_METADATA_CLOSED"); - if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE"); - const owned = this.owners.get(owner) ?? 0; - if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes) - throw Error("CANDIDATE_METADATA_LIMIT"); - bytes += amount; - this.bytes += amount; - this.owners.set(owner, owned + amount); + charge, + /** Opaque, idempotent transient reservation; callers cannot subtract an + * arbitrary amount or release another candidate's charges. */ + reserve: (amount: number) => { + charge(amount); + let released = false; + return () => { + if (closed || released) return; + released = true; + subtract(amount); + }; }, release: () => { if (closed) return; closed = true; - this.bytes -= bytes; + subtract(bytes); this.scopes--; - const remaining = (this.owners.get(owner) ?? 0) - bytes; - if (remaining) this.owners.set(owner, remaining); - else this.owners.delete(owner); }, }; } @@ -53,21 +69,50 @@ export class CandidateMetadataBudget { export type CandidateMetadataScope = ReturnType; export const candidateMetadataBudget = new CandidateMetadataBudget(); +// Estimate JSON allocation without constructing its tree. Object/array headers, +// value slots and strings get explicit weights, with a multiplier for checked +// indexes/derived representations. Indentation must not multiply tree charges. +function jsonAllocation(bytes: Uint8Array): number { + let estimate = 0; + for (let i = 0; i < bytes.length; i++) { + const c = bytes[i]!; + if (c === 123) estimate += 64; + else if (c === 91) estimate += 32; + else if (c === 34) { + const start = ++i; + while (i < bytes.length && bytes[i] !== 34) { + if (bytes[i] === 92) i++; + i++; + } + estimate += 32 + 2 * (i - start); + } else if (c === 45 || (c >= 48 && c <= 57) || c === 116 || c === 102 || c === 110) { + estimate += 16; + while (i + 1 < bytes.length) { + const next = bytes[i + 1]!; + if (next === 9 || next === 10 || next === 13 || next === 32 || next === 44 || next === 93 || next === 125) + break; + i++; + } + } + } + return estimate * 2; +} + /** Immutable regular files only. Fixed allocation plus an extra byte catches * growth; readFile() would allocate before the admission check can intervene. */ export async function readCandidateMetadata( path: string, scope: CandidateMetadataScope, limit: number, -): Promise { +): Promise<{ bytes: Uint8Array; releaseInput: () => void }> { const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let releaseInput: (() => void) | undefined; try { const stat = await handle.stat(); if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size > limit) throw Error("CANDIDATE_METADATA_FILE_LIMIT"); - // Conservative attribution includes strings, small JSON nodes, duplicate - // checked representations, digest serialization and installed indexes. - scope.charge((stat.size + 1) * 32); + // Reserve the input buffer and UTF-16 decoding before allocating either. + releaseInput = scope.reserve((stat.size + 1) * 3); const bytes = Buffer.alloc(stat.size + 1); let offset = 0; while (offset < bytes.length) { @@ -76,7 +121,13 @@ export async function readCandidateMetadata( offset += read.bytesRead; } if (offset !== stat.size) throw Error("CANDIDATE_METADATA_CHANGED"); - return bytes.subarray(0, offset); + const content = bytes.subarray(0, offset); + // No JSON.parse/decoded tree exists until its shape-based charge succeeds. + scope.charge(jsonAllocation(content)); + return { bytes: content, releaseInput }; + } catch (error) { + releaseInput?.(); + throw error; } finally { await handle.close(); } diff --git a/src/execution-world/placement.ts b/src/execution-world/placement.ts index 468e191..1b8fe01 100644 --- a/src/execution-world/placement.ts +++ b/src/execution-world/placement.ts @@ -137,8 +137,9 @@ export function inspectJavaScriptSurface(source: string, checkedSDKImports: read } /** Derived metadata only. Browser/server portability and available targets say - * nothing about sandbox eligibility. No existing package artifact has a - * qualified restricted runtime, so it conservatively requires its VM. */ + * nothing about sandbox eligibility. Build reports have no platform approval + * catalog, so they conservatively require VM placement. A host can override + * placement only for independently reviewed, exact restricted-code approvals. */ export async function reportExecutionPlacement( plan: ExecutionPlan, installation: ExecutionInstallation, diff --git a/test/candidate-metadata.test.ts b/test/candidate-metadata.test.ts index 103dff3..b8871f1 100644 --- a/test/candidate-metadata.test.ts +++ b/test/candidate-metadata.test.ts @@ -28,23 +28,74 @@ test("concurrent candidate reservations enforce aggregate and owner limits and r test("metadata files reserve parse/index space before allocation and refuse oversized or linked inputs", async () => { const root = await mkdtemp(join(tmpdir(), "qx-metadata-budget-")); - const budget = new CandidateMetadataBudget({ bytes: 512, ownerBytes: 512, candidates: 2 }); + const budget = new CandidateMetadataBudget({ bytes: 450, ownerBytes: 450, candidates: 2 }); const scope = budget.acquire("workspace"); try { const file = join(root, "metadata.json"); await writeFile(file, '{"ok":true}'); - assert.equal(new TextDecoder().decode(await readCandidateMetadata(file, scope, 100)), '{"ok":true}'); - assert.equal(scope.bytes, 12 * 32); + const input = await readCandidateMetadata(file, scope, 100); + assert.equal(new TextDecoder().decode(input.bytes), '{"ok":true}'); + const withInput = scope.bytes; + input.releaseInput(); + input.releaseInput(); + const first = scope.bytes; + assert.equal(withInput - first, (Buffer.byteLength('{"ok":true}') + 1) * 3); + assert.ok(first > 12 * 3, "Decoded tree and derived metadata are reserved before parsing"); await assert.rejects(readCandidateMetadata(file, scope, 100), /CANDIDATE_METADATA_LIMIT/); - assert.equal(scope.bytes, 12 * 32); + assert.equal(scope.bytes, first, "Rejected tree allocation releases its transient input buffer"); await assert.rejects(readCandidateMetadata(file, scope, 5), /CANDIDATE_METADATA_FILE_LIMIT/); await symlink(file, join(root, "linked")); await assert.rejects(readCandidateMetadata(join(root, "linked"), scope, 100), { code: "ELOOP" }); await assert.rejects(readCandidateMetadata(root, scope, 100), /CANDIDATE_METADATA_FILE_LIMIT/); - assert.equal(scope.bytes, 12 * 32); + assert.equal(scope.bytes, first); } finally { scope.release(); await rm(root, { recursive: true, force: true }); } assert.equal(budget.snapshot().bytes, 0); }); + +test("JSON structure accounting is independent of indentation and covers strings, values and dense nodes", async () => { + const root = await mkdtemp(join(tmpdir(), "qx-metadata-shape-")); + const budget = new CandidateMetadataBudget(); + try { + for (const value of [{}, [null, true, false, 1, -3.5e8], { 'a"b': '雪\\"', nodes: [{}, {}, {}] }]) { + const measured = []; + for (const spaces of [0, 4]) { + const file = join(root, "metadata.json"), + text = JSON.stringify(value, null, spaces); + await writeFile(file, text); + const scope = budget.acquire("test"); + try { + const input = await readCandidateMetadata(file, scope, 10000); + assert.deepEqual(JSON.parse(new TextDecoder().decode(input.bytes)), value); + input.releaseInput(); + measured.push(scope.bytes); + } finally { + scope.release(); + } + } + assert.equal(measured[0], measured[1], "Whitespace changes input storage, not the allocated JSON shape"); + assert.ok(measured[0] >= 128); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + assert.equal(budget.snapshot().bytes, 0); +}); + +test("transient leases release exactly once, including after their parent closes", () => { + const budget = new CandidateMetadataBudget({ bytes: 1000, ownerBytes: 600, candidates: 3 }); + const scope = budget.acquire("a"); + scope.charge(200); + const release = scope.reserve(300); + assert.throws(() => scope.reserve(101), /CANDIDATE_METADATA_LIMIT/); + release(); + release(); + assert.equal(scope.bytes, 200); + const afterClose = scope.reserve(100); + scope.release(); + afterClose(); + assert.equal(scope.bytes, 0); + assert.equal(budget.snapshot().bytes, 0); +}); From 61534d3fb565601b2b7ac52ef3a651caf27a4870 Mon Sep 17 00:00:00 2001 From: "Timothy J. Aveni" Date: Wed, 7 Oct 2026 16:52:20 -0700 Subject: [PATCH 3/3] Keep checked browser runtime versions independent of authority upgrades --- src/execution-world/candidate.ts | 22 ++++++++++++++++++---- src/execution-world/metadata.ts | 6 +++--- test/candidate-metadata.test.ts | 3 +++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/execution-world/candidate.ts b/src/execution-world/candidate.ts index 090cd57..fd3def5 100644 --- a/src/execution-world/candidate.ts +++ b/src/execution-world/candidate.ts @@ -10,10 +10,16 @@ import { type CandidateMetadataScope, } from "./metadata.js"; const rootPattern = /^\/nix\/store\/[a-z0-9]{32}-[^/\s]+$/; -async function readJson(root: string, file: string, scope: CandidateMetadataScope): Promise { +async function readJson(root: string, file: string, scope: CandidateMetadataScope) { const input = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024); try { - return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(input.bytes)); + return { + value: JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(input.bytes)), + releaseTree: input.releaseTree, + }; + } catch (error) { + input.releaseTree(); + throw error; } finally { input.releaseInput(); } @@ -30,6 +36,7 @@ export async function loadCheckedInstallationCandidate( if (!rootPattern.test(root) || !(await lstat(root)).isDirectory()) throw Error("Candidate must be an immutable Nix output directory"); const scope = (resources.budget ?? candidateMetadataBudget).acquire(resources.owner ?? root); + let releaseValidation = () => {}; try { // Sequential reads avoid partially rejected Promise.all() work allocating // after the failed candidate's reservation has been released. @@ -43,8 +50,13 @@ export async function loadCheckedInstallationCandidate( "kernel.json", "queries.json", "graph-execution.json", - ]) - values.push(await readJson(root, file, scope)); + ]) { + const parsed = await readJson(root, file, scope); + values.push(parsed.value); + // candidate.json duplicates the checked plan for validation. Nothing in + // the installed candidate retains this input tree after validation. + if (file === "candidate.json") releaseValidation = parsed.releaseTree; + } const [candidate, plan, artifacts, source, entryRequest, kernel, queries, graph] = values; const { executionContracts: _legacyContracts, ...workspace } = candidate; if ( @@ -149,6 +161,8 @@ export async function loadCheckedInstallationCandidate( } catch (error) { scope.release(); throw error; + } finally { + releaseValidation(); } } diff --git a/src/execution-world/metadata.ts b/src/execution-world/metadata.ts index a7c0664..66bc5fa 100644 --- a/src/execution-world/metadata.ts +++ b/src/execution-world/metadata.ts @@ -104,7 +104,7 @@ export async function readCandidateMetadata( path: string, scope: CandidateMetadataScope, limit: number, -): Promise<{ bytes: Uint8Array; releaseInput: () => void }> { +): Promise<{ bytes: Uint8Array; releaseInput: () => void; releaseTree: () => void }> { const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); let releaseInput: (() => void) | undefined; try { @@ -123,8 +123,8 @@ export async function readCandidateMetadata( if (offset !== stat.size) throw Error("CANDIDATE_METADATA_CHANGED"); const content = bytes.subarray(0, offset); // No JSON.parse/decoded tree exists until its shape-based charge succeeds. - scope.charge(jsonAllocation(content)); - return { bytes: content, releaseInput }; + const releaseTree = scope.reserve(jsonAllocation(content)); + return { bytes: content, releaseInput, releaseTree }; } catch (error) { releaseInput?.(); throw error; diff --git a/test/candidate-metadata.test.ts b/test/candidate-metadata.test.ts index b8871f1..67e96f9 100644 --- a/test/candidate-metadata.test.ts +++ b/test/candidate-metadata.test.ts @@ -48,6 +48,9 @@ test("metadata files reserve parse/index space before allocation and refuse over await assert.rejects(readCandidateMetadata(join(root, "linked"), scope, 100), { code: "ELOOP" }); await assert.rejects(readCandidateMetadata(root, scope, 100), /CANDIDATE_METADATA_FILE_LIMIT/); assert.equal(scope.bytes, first); + input.releaseTree(); + input.releaseTree(); + assert.equal(scope.bytes, 0, "Discarded validation trees can release independently of the candidate"); } finally { scope.release(); await rm(root, { recursive: true, force: true });