diff --git a/src/execution-world/candidate.ts b/src/execution-world/candidate.ts index 1be6d37..171a5fb 100644 --- a/src/execution-world/candidate.ts +++ b/src/execution-world/candidate.ts @@ -1,19 +1,18 @@ import { graphExecutionRepresentation } from "./graph-execution.js"; -import { open, lstat } from "node:fs/promises"; -import { constants } from "node:fs"; +import { lstat } from "node:fs/promises"; import { join } from "node:path"; import { hexDigest, ExecutionPlanError, type ExecutionPlan } from "./plan.js"; import { loadExecutionInstallation, type PackageInstallationSelection } from "./installation.js"; +import { + candidateMetadataBudget, + readCandidateMetadata, + type CandidateMetadataBudget, + type CandidateMetadataScope, +} from "./metadata.js"; const rootPattern = /^\/nix\/store\/[a-z0-9]{32}-[^/\s]+$/; -async function readJson(root: string, file: string): Promise { - const handle = await open(join(root, file), constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const metadata = await handle.stat(); - if (!metadata.isFile() || metadata.size > 32 * 1024 * 1024) throw Error(`Invalid checked candidate file: ${file}`); - return JSON.parse(await handle.readFile("utf8")); - } finally { - await handle.close(); - } +async function readJson(root: string, file: string, scope: CandidateMetadataScope): Promise { + const bytes = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024); + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); } /** Load the checked Nix output, preserving its original compilation and provenance. * The selected toolchain validates generated code during the build. This loader @@ -22,113 +21,131 @@ async function readJson(root: string, file: string): Promise { export async function loadCheckedInstallationCandidate( root: string, selections: Readonly> = {}, + resources: { budget?: CandidateMetadataBudget; owner?: string } = {}, ) { if (!rootPattern.test(root) || !(await lstat(root)).isDirectory()) throw Error("Candidate must be an immutable Nix output directory"); - const [candidate, plan, artifacts, source, entryRequest, kernel, queries, graph] = await Promise.all([ - readJson(root, "candidate.json"), - readJson(root, "execution-plan.json"), - readJson(root, "execution-artifacts.json"), - readJson(root, "source.json"), - readJson(root, "entry.json"), - readJson(root, "kernel.json"), - readJson(root, "queries.json"), - readJson(root, "graph-execution.json"), - ]); - const { executionContracts: _legacyContracts, ...workspace } = candidate; - if ( - plan?.definition?.codec !== "camino-execution-plan-v1" || - hexDigest(plan.definition) !== plan.digest || - hexDigest(workspace) !== hexDigest(plan.definition.workspace) - ) - throw new ExecutionPlanError( - "CANDIDATE_PLAN_MISMATCH", - root, - "Stored plan must match its digest and exact workspace source contract", - ); - if ( - source.kind !== "workspace" || - source.commit !== plan.definition.workspace.sourceRootCommit || - typeof source.repository !== "string" - ) - throw new ExecutionPlanError("CANDIDATE_SOURCE_MISMATCH", root, "Expected exact workspace source provenance"); - const repository = new URL(source.repository); - if ( - !["https:", "ssh:"].includes(repository.protocol) || - repository.username || - repository.password || - repository.hash || - repository.search - ) - throw new ExecutionPlanError("CANDIDATE_SOURCE_INVALID", root, "Expected a source URL without credentials"); - if ( - artifacts.codec !== "camino-checked-execution-v2" || - typeof artifacts.generator !== "string" || - !/^[0-9a-f]{64}$/.test(artifacts.generator) || - !artifacts.roots || - typeof artifacts.roots !== "object" || - Array.isArray(artifacts.roots) - ) - throw new ExecutionPlanError("CANDIDATE_ARTIFACTS_INVALID", root, "Expected checked package roots"); - for (const entry of Object.values(artifacts.roots)) - if (typeof entry !== "string" || !rootPattern.test(entry)) + const scope = (resources.budget ?? candidateMetadataBudget).acquire(resources.owner ?? root); + try { + // Sequential reads avoid partially rejected Promise.all() work allocating + // after the failed candidate's reservation has been released. + const values = []; + for (const file of [ + "candidate.json", + "execution-plan.json", + "execution-artifacts.json", + "source.json", + "entry.json", + "kernel.json", + "queries.json", + "graph-execution.json", + ]) + values.push(await readJson(root, file, scope)); + const [candidate, plan, artifacts, source, entryRequest, kernel, queries, graph] = values; + const { executionContracts: _legacyContracts, ...workspace } = candidate; + if ( + plan?.definition?.codec !== "camino-execution-plan-v1" || + hexDigest(plan.definition) !== plan.digest || + hexDigest(workspace) !== hexDigest(plan.definition.workspace) + ) throw new ExecutionPlanError( - "CANDIDATE_ARTIFACTS_INVALID", + "CANDIDATE_PLAN_MISMATCH", root, - "Package artifacts must be immutable Nix roots", + "Stored plan must match its digest and exact workspace source contract", ); - const installation = await loadExecutionInstallation(plan, artifacts.roots, selections); - const entry = compileWorkspaceEntry(plan, entryRequest); - for (const [name, program, codec] of [ - ["kernel", kernel, "camino-kernel-program-v1"], - ["queries", queries, "camino-query-program-v1"], - ] as const) - if (program?.codec !== codec || program.plan !== kernel.plan || hexDigest(program) !== artifacts.programs?.[name]) + if ( + source.kind !== "workspace" || + source.commit !== plan.definition.workspace.sourceRootCommit || + typeof source.repository !== "string" + ) + throw new ExecutionPlanError("CANDIDATE_SOURCE_MISMATCH", root, "Expected exact workspace source provenance"); + const repository = new URL(source.repository); + if ( + !["https:", "ssh:"].includes(repository.protocol) || + repository.username || + repository.password || + repository.hash || + repository.search + ) + throw new ExecutionPlanError("CANDIDATE_SOURCE_INVALID", root, "Expected a source URL without credentials"); + if ( + artifacts.codec !== "camino-checked-execution-v2" || + typeof artifacts.generator !== "string" || + !/^[0-9a-f]{64}$/.test(artifacts.generator) || + !artifacts.roots || + typeof artifacts.roots !== "object" || + Array.isArray(artifacts.roots) + ) + throw new ExecutionPlanError("CANDIDATE_ARTIFACTS_INVALID", root, "Expected checked package roots"); + for (const entry of Object.values(artifacts.roots)) + if (typeof entry !== "string" || !rootPattern.test(entry)) + throw new ExecutionPlanError( + "CANDIDATE_ARTIFACTS_INVALID", + root, + "Package artifacts must be immutable Nix roots", + ); + const installation = await loadExecutionInstallation(plan, artifacts.roots, selections, scope); + const entry = compileWorkspaceEntry(plan, entryRequest); + for (const [name, program, codec] of [ + ["kernel", kernel, "camino-kernel-program-v1"], + ["queries", queries, "camino-query-program-v1"], + ] as const) + if (program?.codec !== codec || program.plan !== kernel.plan || hexDigest(program) !== artifacts.programs?.[name]) + throw new ExecutionPlanError( + "CANDIDATE_PROGRAM_MISMATCH", + name, + "Stored interpreter input must match the checked build and plan", + ); + const expectedGraph = graphExecutionRepresentation(plan, installation, kernel, queries); + if (hexDigest(graph) !== hexDigest(expectedGraph) || artifacts.execution !== hexDigest(graph)) throw new ExecutionPlanError( - "CANDIDATE_PROGRAM_MISMATCH", - name, - "Stored interpreter input must match the checked build and plan", + "CANDIDATE_EXECUTION_MISMATCH", + root, + "Stored graph execution differs from checked inputs", ); - const expectedGraph = graphExecutionRepresentation(plan, installation, kernel, queries); - if (hexDigest(graph) !== hexDigest(expectedGraph) || artifacts.execution !== hexDigest(graph)) - throw new ExecutionPlanError( - "CANDIDATE_EXECUTION_MISMATCH", + const provenance = { + workspaceRevision: workspace.id as string, + source: { repository: source.repository as string, commit: source.commit as string }, + generator: artifacts.generator as string, + packages: (plan as ExecutionPlan).definition.packages.map((pkg) => ({ + revision: pkg.revision, + source: pkg.source, + })), + }; + const reference = { root, - "Stored graph execution differs from checked inputs", - ); - const provenance = { - workspaceRevision: workspace.id as string, - source: { repository: source.repository as string, commit: source.commit as string }, - generator: artifacts.generator as string, - packages: (plan as ExecutionPlan).definition.packages.map((pkg) => ({ - revision: pkg.revision, - source: pkg.source, - })), - }; - const reference = { - root, - digest: hexDigest({ - plan: plan.digest, - world: installation.world, + digest: hexDigest({ + plan: plan.digest, + world: installation.world, + provenance, + roots: artifacts.roots, + entry, + programs: artifacts.programs, + }), + }; + return { + reference, + workspace: plan.definition.workspace.workspaceId, + world: hexDigest(graph), + plan: kernel.plan as string, + graphExecution: graph, provenance, - roots: artifacts.roots, entry, - programs: artifacts.programs, - }), - }; - return { - reference, - workspace: plan.definition.workspace.workspaceId, - world: hexDigest(graph), - plan: kernel.plan as string, - graphExecution: graph, - provenance, - entry, - kernel, - queries, - installation, - executionPlan: plan as ExecutionPlan, - }; + kernel, + queries, + installation, + executionPlan: plan as ExecutionPlan, + metadata: { + get bytes() { + return scope.bytes; + }, + release: scope.release, + }, + }; + } catch (error) { + scope.release(); + throw error; + } } export function compileWorkspaceEntry(plan: ExecutionPlan, entryRequest: unknown): { operation: string } | null { diff --git a/src/execution-world/files.ts b/src/execution-world/files.ts index f5b627a..4c59a09 100644 --- a/src/execution-world/files.ts +++ b/src/execution-world/files.ts @@ -13,6 +13,7 @@ import { type CheckedWorld, } from "./artifacts.js"; import { type ExecutionPlan, ExecutionPlanError } from "./plan.js"; +import type { CandidateMetadataScope } from "./metadata.js"; const maxFileBytes = 256 * 1024 * 1024; const metadataLimit = 16 * 1024 * 1024; @@ -21,7 +22,10 @@ const fail = (where: string, message: string): never => { }; /** Roots must be trusted immutable output directories. Refuse symlinks, including * interior directory links. Hash verification is still required before serving bytes. */ -export async function artifactFileReader(roots: Readonly>): Promise { +export async function artifactFileReader( + roots: Readonly>, + metadata?: CandidateMetadataScope, +): Promise { const directories = new Map(); for (const [revision, root] of Object.entries(roots)) { if (!path.isAbsolute(root) || path.normalize(root) !== root || (await realpath(root)) !== root) @@ -47,6 +51,7 @@ export async function artifactFileReader(roots: Readonly> try { const stat = await handle.stat(); if (!stat.isFile() || stat.size > limit) fail(relative, "File exceeds its declared bound"); + if (relative === "share/quixos/package-artifacts.json") metadata?.charge((stat.size + 1) * 32); // A fixed allocation and one extra byte detect growth without readFile's unbounded allocation. const buffer = Buffer.alloc(stat.size + 1); let offset = 0; @@ -72,8 +77,9 @@ export interface LoadedArtifactWorld { export async function loadArtifactWorld( plan: ExecutionPlan, roots: Readonly>, + metadata?: CandidateMetadataScope, ): Promise { - const read = await artifactFileReader(roots), + const read = await artifactFileReader(roots, metadata), manifests: PackageArtifacts[] = []; const expected = plan.definition.packages.map((p) => p.revision).sort(); if (JSON.stringify(Object.keys(roots).sort()) !== JSON.stringify(expected)) diff --git a/src/execution-world/index.ts b/src/execution-world/index.ts index a8e0b00..f605a24 100644 --- a/src/execution-world/index.ts +++ b/src/execution-world/index.ts @@ -17,6 +17,7 @@ export * from "./queries.js"; export * from "./query-document.js"; export { generatePortableClientBindings } from "../bindings/portable-client.js"; export * from "./candidate.js"; +export * from "./metadata.js"; export { reviewSemanticMajors } from "../capability-model/evolution.js"; diff --git a/src/execution-world/installation.ts b/src/execution-world/installation.ts index 61be4dc..3c9d26d 100644 --- a/src/execution-world/installation.ts +++ b/src/execution-world/installation.ts @@ -11,6 +11,7 @@ import { type PublicQueryContract, } from "./client-contracts.js"; import { resultInterfaces } from "./projections.js"; +import type { CandidateMetadataScope } from "./metadata.js"; /** Activation is assigned by graph authority, never by a package artifact. */ export interface ActiveExecutionIdentity { @@ -159,8 +160,9 @@ export async function loadExecutionInstallation( plan: ExecutionPlan, roots: Readonly>, selections: Readonly> = {}, + metadata?: CandidateMetadataScope, ): Promise { - return assembleExecutionInstallation(plan, roots, await loadArtifactWorld(plan, roots), selections); + return assembleExecutionInstallation(plan, roots, await loadArtifactWorld(plan, roots, metadata), selections); } function assembleExecutionInstallation( plan: ExecutionPlan, diff --git a/src/execution-world/metadata.ts b/src/execution-world/metadata.ts new file mode 100644 index 0000000..847b993 --- /dev/null +++ b/src/execution-world/metadata.ts @@ -0,0 +1,83 @@ +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; + +/** Admission estimates, not a hard V8 heap limit. A process/cgroup ceiling is + * still required. Charge input, parse trees and derived indexes before reading + * or parsing. Each live candidate retains its reservation until explicit close. */ +export class CandidateMetadataBudget { + readonly limits: Readonly<{ bytes: number; ownerBytes: number; candidates: number }>; + private bytes = 0; + private scopes = 0; + private readonly owners = new Map(); + constructor(limits = { bytes: 512 * 1024 * 1024, ownerBytes: 256 * 1024 * 1024, candidates: 128 }) { + for (const value of Object.values(limits)) + if (!Number.isSafeInteger(value) || value < 1) throw Error("INVALID_CANDIDATE_METADATA_BUDGET"); + if (limits.ownerBytes > limits.bytes) throw Error("INVALID_CANDIDATE_METADATA_BUDGET"); + this.limits = Object.freeze({ ...limits }); + } + snapshot() { + return { bytes: this.bytes, candidates: this.scopes, owners: this.owners.size, limits: this.limits }; + } + acquire(owner: string) { + if (!owner || owner.length > 4096) throw Error("INVALID_CANDIDATE_METADATA_OWNER"); + if (this.scopes >= this.limits.candidates) throw Error("CANDIDATE_METADATA_LIMIT"); + this.scopes++; + let bytes = 0, + closed = false; + return { + get bytes() { + return bytes; + }, + charge: (amount: number) => { + if (closed) throw Error("CANDIDATE_METADATA_CLOSED"); + if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE"); + const owned = this.owners.get(owner) ?? 0; + if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes) + throw Error("CANDIDATE_METADATA_LIMIT"); + bytes += amount; + this.bytes += amount; + this.owners.set(owner, owned + amount); + }, + release: () => { + if (closed) return; + closed = true; + this.bytes -= bytes; + this.scopes--; + const remaining = (this.owners.get(owner) ?? 0) - bytes; + if (remaining) this.owners.set(owner, remaining); + else this.owners.delete(owner); + }, + }; + } +} +export type CandidateMetadataScope = ReturnType; +export const candidateMetadataBudget = new CandidateMetadataBudget(); + +/** Immutable regular files only. Fixed allocation plus an extra byte catches + * growth; readFile() would allocate before the admission check can intervene. */ +export async function readCandidateMetadata( + path: string, + scope: CandidateMetadataScope, + limit: number, +): Promise { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size > limit) + throw Error("CANDIDATE_METADATA_FILE_LIMIT"); + // Conservative attribution includes strings, small JSON nodes, duplicate + // checked representations, digest serialization and installed indexes. + scope.charge((stat.size + 1) * 32); + const bytes = Buffer.alloc(stat.size + 1); + let offset = 0; + while (offset < bytes.length) { + const read = await handle.read(bytes, offset, bytes.length - offset, offset); + if (!read.bytesRead) break; + offset += read.bytesRead; + } + if (offset !== stat.size) throw Error("CANDIDATE_METADATA_CHANGED"); + return bytes.subarray(0, offset); + } finally { + await handle.close(); + } +} diff --git a/src/execution-world/ports.ts b/src/execution-world/ports.ts index d26e5a3..4a709bc 100644 --- a/src/execution-world/ports.ts +++ b/src/execution-world/ports.ts @@ -29,6 +29,8 @@ export interface PackageInvocation { /** Capability arguments/receiver use the versioned managed-reference codec. */ input: unknown; receiver: unknown | null; + /** Host-owned remaining root resources, separate from authored input. */ + budget?: { root: string; depth: number; calls: number; deadlineMs: number }; } export type InvocationReply = | { abi: typeof transactionPortAbi; invocation: string; status: "returned"; output: unknown } diff --git a/test/candidate-metadata.test.ts b/test/candidate-metadata.test.ts new file mode 100644 index 0000000..103dff3 --- /dev/null +++ b/test/candidate-metadata.test.ts @@ -0,0 +1,50 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile, symlink, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { CandidateMetadataBudget, readCandidateMetadata } from "../src/execution-world/metadata.js"; + +test("concurrent candidate reservations enforce aggregate and owner limits and release exactly once", () => { + const budget = new CandidateMetadataBudget({ bytes: 1000, ownerBytes: 600, candidates: 3 }); + const first = budget.acquire("a"), + second = budget.acquire("a"), + third = budget.acquire("b"); + assert.throws(() => budget.acquire("c"), /CANDIDATE_METADATA_LIMIT/); + first.charge(400); + second.charge(200); + third.charge(400); + assert.throws(() => second.charge(1), /CANDIDATE_METADATA_LIMIT/); + assert.throws(() => third.charge(1), /CANDIDATE_METADATA_LIMIT/); + assert.equal(budget.snapshot().bytes, 1000); + second.release(); + second.release(); + assert.equal(budget.snapshot().bytes, 800); + assert.throws(() => second.charge(1), /CANDIDATE_METADATA_CLOSED/); + first.release(); + third.release(); + assert.deepEqual(budget.snapshot(), { bytes: 0, candidates: 0, owners: 0, limits: budget.limits }); +}); + +test("metadata files reserve parse/index space before allocation and refuse oversized or linked inputs", async () => { + const root = await mkdtemp(join(tmpdir(), "qx-metadata-budget-")); + const budget = new CandidateMetadataBudget({ bytes: 512, ownerBytes: 512, candidates: 2 }); + const scope = budget.acquire("workspace"); + try { + const file = join(root, "metadata.json"); + await writeFile(file, '{"ok":true}'); + assert.equal(new TextDecoder().decode(await readCandidateMetadata(file, scope, 100)), '{"ok":true}'); + assert.equal(scope.bytes, 12 * 32); + await assert.rejects(readCandidateMetadata(file, scope, 100), /CANDIDATE_METADATA_LIMIT/); + assert.equal(scope.bytes, 12 * 32); + await assert.rejects(readCandidateMetadata(file, scope, 5), /CANDIDATE_METADATA_FILE_LIMIT/); + await symlink(file, join(root, "linked")); + await assert.rejects(readCandidateMetadata(join(root, "linked"), scope, 100), { code: "ELOOP" }); + await assert.rejects(readCandidateMetadata(root, scope, 100), /CANDIDATE_METADATA_FILE_LIMIT/); + assert.equal(scope.bytes, 12 * 32); + } finally { + scope.release(); + await rm(root, { recursive: true, force: true }); + } + assert.equal(budget.snapshot().bytes, 0); +});