Bound checked candidate metadata and restricted invocation resources

src/execution-world/candidate.ts

@@ -1,19 +1,18 @@
 import { graphExecutionRepresentation } from "./graph-execution.js";
-import { open, lstat } from "node:fs/promises";
-import { constants } from "node:fs";
+import { lstat } from "node:fs/promises";
 import { join } from "node:path";
 import { hexDigest, ExecutionPlanError, type ExecutionPlan } from "./plan.js";
 import { loadExecutionInstallation, type PackageInstallationSelection } from "./installation.js";
+import {
+  candidateMetadataBudget,
+  readCandidateMetadata,
+  type CandidateMetadataBudget,
+  type CandidateMetadataScope,
+} from "./metadata.js";
 const rootPattern = /^\/nix\/store\/[a-z0-9]{32}-[^/\s]+$/;
-async function readJson(root: string, file: string): Promise<any> {
-  const handle = await open(join(root, file), constants.O_RDONLY | constants.O_NOFOLLOW);
-  try {
-    const metadata = await handle.stat();
-    if (!metadata.isFile() || metadata.size > 32 * 1024 * 1024) throw Error(`Invalid checked candidate file: ${file}`);
-    return JSON.parse(await handle.readFile("utf8"));
-  } finally {
-    await handle.close();
-  }
+async function readJson(root: string, file: string, scope: CandidateMetadataScope): Promise<any> {
+  const bytes = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024);
+  return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
 }
 /** Load the checked Nix output, preserving its original compilation and provenance.
  * The selected toolchain validates generated code during the build. This loader
@@ -22,113 +21,131 @@ async function readJson(root: string, file: string): Promise<any> {
 export async function loadCheckedInstallationCandidate(
   root: string,
   selections: Readonly<Record<string, PackageInstallationSelection>> = {},
+  resources: { budget?: CandidateMetadataBudget; owner?: string } = {},
 ) {
   if (!rootPattern.test(root) || !(await lstat(root)).isDirectory())
     throw Error("Candidate must be an immutable Nix output directory");
-  const [candidate, plan, artifacts, source, entryRequest, kernel, queries, graph] = await Promise.all([
-    readJson(root, "candidate.json"),
-    readJson(root, "execution-plan.json"),
-    readJson(root, "execution-artifacts.json"),
-    readJson(root, "source.json"),
-    readJson(root, "entry.json"),
-    readJson(root, "kernel.json"),
-    readJson(root, "queries.json"),
-    readJson(root, "graph-execution.json"),
-  ]);
-  const { executionContracts: _legacyContracts, ...workspace } = candidate;
-  if (
-    plan?.definition?.codec !== "camino-execution-plan-v1" ||
-    hexDigest(plan.definition) !== plan.digest ||
-    hexDigest(workspace) !== hexDigest(plan.definition.workspace)
-  )
-    throw new ExecutionPlanError(
-      "CANDIDATE_PLAN_MISMATCH",
-      root,
-      "Stored plan must match its digest and exact workspace source contract",
-    );
-  if (
-    source.kind !== "workspace" ||
-    source.commit !== plan.definition.workspace.sourceRootCommit ||
-    typeof source.repository !== "string"
-  )
-    throw new ExecutionPlanError("CANDIDATE_SOURCE_MISMATCH", root, "Expected exact workspace source provenance");
-  const repository = new URL(source.repository);
-  if (
-    !["https:", "ssh:"].includes(repository.protocol) ||
-    repository.username ||
-    repository.password ||
-    repository.hash ||
-    repository.search
-  )
-    throw new ExecutionPlanError("CANDIDATE_SOURCE_INVALID", root, "Expected a source URL without credentials");
-  if (
-    artifacts.codec !== "camino-checked-execution-v2" ||
-    typeof artifacts.generator !== "string" ||
-    !/^[0-9a-f]{64}$/.test(artifacts.generator) ||
-    !artifacts.roots ||
-    typeof artifacts.roots !== "object" ||
-    Array.isArray(artifacts.roots)
-  )
-    throw new ExecutionPlanError("CANDIDATE_ARTIFACTS_INVALID", root, "Expected checked package roots");
-  for (const entry of Object.values(artifacts.roots))
-    if (typeof entry !== "string" || !rootPattern.test(entry))
+  const scope = (resources.budget ?? candidateMetadataBudget).acquire(resources.owner ?? root);
+  try {
+    // Sequential reads avoid partially rejected Promise.all() work allocating
+    // after the failed candidate's reservation has been released.
+    const values = [];
+    for (const file of [
+      "candidate.json",
+      "execution-plan.json",
+      "execution-artifacts.json",
+      "source.json",
+      "entry.json",
+      "kernel.json",
+      "queries.json",
+      "graph-execution.json",
+    ])
+      values.push(await readJson(root, file, scope));
+    const [candidate, plan, artifacts, source, entryRequest, kernel, queries, graph] = values;
+    const { executionContracts: _legacyContracts, ...workspace } = candidate;
+    if (
+      plan?.definition?.codec !== "camino-execution-plan-v1" ||
+      hexDigest(plan.definition) !== plan.digest ||
+      hexDigest(workspace) !== hexDigest(plan.definition.workspace)
+    )
       throw new ExecutionPlanError(
-        "CANDIDATE_ARTIFACTS_INVALID",
+        "CANDIDATE_PLAN_MISMATCH",
         root,
-        "Package artifacts must be immutable Nix roots",
+        "Stored plan must match its digest and exact workspace source contract",
       );
-  const installation = await loadExecutionInstallation(plan, artifacts.roots, selections);
-  const entry = compileWorkspaceEntry(plan, entryRequest);
-  for (const [name, program, codec] of [
-    ["kernel", kernel, "camino-kernel-program-v1"],
-    ["queries", queries, "camino-query-program-v1"],
-  ] as const)
-    if (program?.codec !== codec || program.plan !== kernel.plan || hexDigest(program) !== artifacts.programs?.[name])
+    if (
+      source.kind !== "workspace" ||
+      source.commit !== plan.definition.workspace.sourceRootCommit ||
+      typeof source.repository !== "string"
+    )
+      throw new ExecutionPlanError("CANDIDATE_SOURCE_MISMATCH", root, "Expected exact workspace source provenance");
+    const repository = new URL(source.repository);
+    if (
+      !["https:", "ssh:"].includes(repository.protocol) ||
+      repository.username ||
+      repository.password ||
+      repository.hash ||
+      repository.search
+    )
+      throw new ExecutionPlanError("CANDIDATE_SOURCE_INVALID", root, "Expected a source URL without credentials");
+    if (
+      artifacts.codec !== "camino-checked-execution-v2" ||
+      typeof artifacts.generator !== "string" ||
+      !/^[0-9a-f]{64}$/.test(artifacts.generator) ||
+      !artifacts.roots ||
+      typeof artifacts.roots !== "object" ||
+      Array.isArray(artifacts.roots)
+    )
+      throw new ExecutionPlanError("CANDIDATE_ARTIFACTS_INVALID", root, "Expected checked package roots");
+    for (const entry of Object.values(artifacts.roots))
+      if (typeof entry !== "string" || !rootPattern.test(entry))
+        throw new ExecutionPlanError(
+          "CANDIDATE_ARTIFACTS_INVALID",
+          root,
+          "Package artifacts must be immutable Nix roots",
+        );
+    const installation = await loadExecutionInstallation(plan, artifacts.roots, selections, scope);
+    const entry = compileWorkspaceEntry(plan, entryRequest);
+    for (const [name, program, codec] of [
+      ["kernel", kernel, "camino-kernel-program-v1"],
+      ["queries", queries, "camino-query-program-v1"],
+    ] as const)
+      if (program?.codec !== codec || program.plan !== kernel.plan || hexDigest(program) !== artifacts.programs?.[name])
+        throw new ExecutionPlanError(
+          "CANDIDATE_PROGRAM_MISMATCH",
+          name,
+          "Stored interpreter input must match the checked build and plan",
+        );
+    const expectedGraph = graphExecutionRepresentation(plan, installation, kernel, queries);
+    if (hexDigest(graph) !== hexDigest(expectedGraph) || artifacts.execution !== hexDigest(graph))
       throw new ExecutionPlanError(
-        "CANDIDATE_PROGRAM_MISMATCH",
-        name,
-        "Stored interpreter input must match the checked build and plan",
+        "CANDIDATE_EXECUTION_MISMATCH",
+        root,
+        "Stored graph execution differs from checked inputs",
       );
-  const expectedGraph = graphExecutionRepresentation(plan, installation, kernel, queries);
-  if (hexDigest(graph) !== hexDigest(expectedGraph) || artifacts.execution !== hexDigest(graph))
-    throw new ExecutionPlanError(
-      "CANDIDATE_EXECUTION_MISMATCH",
+    const provenance = {
+      workspaceRevision: workspace.id as string,
+      source: { repository: source.repository as string, commit: source.commit as string },
+      generator: artifacts.generator as string,
+      packages: (plan as ExecutionPlan).definition.packages.map((pkg) => ({
+        revision: pkg.revision,
+        source: pkg.source,
+      })),
+    };
+    const reference = {
       root,
-      "Stored graph execution differs from checked inputs",
-    );
-  const provenance = {
-    workspaceRevision: workspace.id as string,
-    source: { repository: source.repository as string, commit: source.commit as string },
-    generator: artifacts.generator as string,
-    packages: (plan as ExecutionPlan).definition.packages.map((pkg) => ({
-      revision: pkg.revision,
-      source: pkg.source,
-    })),
-  };
-  const reference = {
-    root,
-    digest: hexDigest({
-      plan: plan.digest,
-      world: installation.world,
+      digest: hexDigest({
+        plan: plan.digest,
+        world: installation.world,
+        provenance,
+        roots: artifacts.roots,
+        entry,
+        programs: artifacts.programs,
+      }),
+    };
+    return {
+      reference,
+      workspace: plan.definition.workspace.workspaceId,
+      world: hexDigest(graph),
+      plan: kernel.plan as string,
+      graphExecution: graph,
       provenance,
-      roots: artifacts.roots,
       entry,
-      programs: artifacts.programs,
-    }),
-  };
-  return {
-    reference,
-    workspace: plan.definition.workspace.workspaceId,
-    world: hexDigest(graph),
-    plan: kernel.plan as string,
-    graphExecution: graph,
-    provenance,
-    entry,
-    kernel,
-    queries,
-    installation,
-    executionPlan: plan as ExecutionPlan,
-  };
+      kernel,
+      queries,
+      installation,
+      executionPlan: plan as ExecutionPlan,
+      metadata: {
+        get bytes() {
+          return scope.bytes;
+        },
+        release: scope.release,
+      },
+    };
+  } catch (error) {
+    scope.release();
+    throw error;
+  }
 }
 
 export function compileWorkspaceEntry(plan: ExecutionPlan, entryRequest: unknown): { operation: string } | null {

src/execution-world/files.ts

@@ -13,6 +13,7 @@ import {
   type CheckedWorld,
 } from "./artifacts.js";
 import { type ExecutionPlan, ExecutionPlanError } from "./plan.js";
+import type { CandidateMetadataScope } from "./metadata.js";
 
 const maxFileBytes = 256 * 1024 * 1024;
 const metadataLimit = 16 * 1024 * 1024;
@@ -21,7 +22,10 @@ const fail = (where: string, message: string): never => {
 };
 /** Roots must be trusted immutable output directories. Refuse symlinks, including
  * interior directory links. Hash verification is still required before serving bytes. */
-export async function artifactFileReader(roots: Readonly<Record<string, string>>): Promise<ArtifactReader> {
+export async function artifactFileReader(
+  roots: Readonly<Record<string, string>>,
+  metadata?: CandidateMetadataScope,
+): Promise<ArtifactReader> {
   const directories = new Map<string, string>();
   for (const [revision, root] of Object.entries(roots)) {
     if (!path.isAbsolute(root) || path.normalize(root) !== root || (await realpath(root)) !== root)
@@ -47,6 +51,7 @@ export async function artifactFileReader(roots: Readonly<Record<string, string>>
     try {
       const stat = await handle.stat();
       if (!stat.isFile() || stat.size > limit) fail(relative, "File exceeds its declared bound");
+      if (relative === "share/quixos/package-artifacts.json") metadata?.charge((stat.size + 1) * 32);
       // A fixed allocation and one extra byte detect growth without readFile's unbounded allocation.
       const buffer = Buffer.alloc(stat.size + 1);
       let offset = 0;
@@ -72,8 +77,9 @@ export interface LoadedArtifactWorld {
 export async function loadArtifactWorld(
   plan: ExecutionPlan,
   roots: Readonly<Record<string, string>>,
+  metadata?: CandidateMetadataScope,
 ): Promise<LoadedArtifactWorld> {
-  const read = await artifactFileReader(roots),
+  const read = await artifactFileReader(roots, metadata),
     manifests: PackageArtifacts[] = [];
   const expected = plan.definition.packages.map((p) => p.revision).sort();
   if (JSON.stringify(Object.keys(roots).sort()) !== JSON.stringify(expected))

src/execution-world/index.ts

@@ -17,6 +17,7 @@ export * from "./queries.js";
 export * from "./query-document.js";
 export { generatePortableClientBindings } from "../bindings/portable-client.js";
 export * from "./candidate.js";
+export * from "./metadata.js";
 
 export { reviewSemanticMajors } from "../capability-model/evolution.js";
 

src/execution-world/installation.ts

@@ -11,6 +11,7 @@ import {
   type PublicQueryContract,
 } from "./client-contracts.js";
 import { resultInterfaces } from "./projections.js";
+import type { CandidateMetadataScope } from "./metadata.js";
 
 /** Activation is assigned by graph authority, never by a package artifact. */
 export interface ActiveExecutionIdentity {
@@ -159,8 +160,9 @@ export async function loadExecutionInstallation(
   plan: ExecutionPlan,
   roots: Readonly<Record<string, string>>,
   selections: Readonly<Record<string, PackageInstallationSelection>> = {},
+  metadata?: CandidateMetadataScope,
 ): Promise<ExecutionInstallation> {
-  return assembleExecutionInstallation(plan, roots, await loadArtifactWorld(plan, roots), selections);
+  return assembleExecutionInstallation(plan, roots, await loadArtifactWorld(plan, roots, metadata), selections);
 }
 function assembleExecutionInstallation(
   plan: ExecutionPlan,

src/execution-world/metadata.ts

@@ -0,0 +1,83 @@
+import { constants } from "node:fs";
+import { open } from "node:fs/promises";
+
+/** Admission estimates, not a hard V8 heap limit. A process/cgroup ceiling is
+ * still required. Charge input, parse trees and derived indexes before reading
+ * or parsing. Each live candidate retains its reservation until explicit close. */
+export class CandidateMetadataBudget {
+  readonly limits: Readonly<{ bytes: number; ownerBytes: number; candidates: number }>;
+  private bytes = 0;
+  private scopes = 0;
+  private readonly owners = new Map<string, number>();
+  constructor(limits = { bytes: 512 * 1024 * 1024, ownerBytes: 256 * 1024 * 1024, candidates: 128 }) {
+    for (const value of Object.values(limits))
+      if (!Number.isSafeInteger(value) || value < 1) throw Error("INVALID_CANDIDATE_METADATA_BUDGET");
+    if (limits.ownerBytes > limits.bytes) throw Error("INVALID_CANDIDATE_METADATA_BUDGET");
+    this.limits = Object.freeze({ ...limits });
+  }
+  snapshot() {
+    return { bytes: this.bytes, candidates: this.scopes, owners: this.owners.size, limits: this.limits };
+  }
+  acquire(owner: string) {
+    if (!owner || owner.length > 4096) throw Error("INVALID_CANDIDATE_METADATA_OWNER");
+    if (this.scopes >= this.limits.candidates) throw Error("CANDIDATE_METADATA_LIMIT");
+    this.scopes++;
+    let bytes = 0,
+      closed = false;
+    return {
+      get bytes() {
+        return bytes;
+      },
+      charge: (amount: number) => {
+        if (closed) throw Error("CANDIDATE_METADATA_CLOSED");
+        if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE");
+        const owned = this.owners.get(owner) ?? 0;
+        if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes)
+          throw Error("CANDIDATE_METADATA_LIMIT");
+        bytes += amount;
+        this.bytes += amount;
+        this.owners.set(owner, owned + amount);
+      },
+      release: () => {
+        if (closed) return;
+        closed = true;
+        this.bytes -= bytes;
+        this.scopes--;
+        const remaining = (this.owners.get(owner) ?? 0) - bytes;
+        if (remaining) this.owners.set(owner, remaining);
+        else this.owners.delete(owner);
+      },
+    };
+  }
+}
+export type CandidateMetadataScope = ReturnType<CandidateMetadataBudget["acquire"]>;
+export const candidateMetadataBudget = new CandidateMetadataBudget();
+
+/** Immutable regular files only. Fixed allocation plus an extra byte catches
+ * growth; readFile() would allocate before the admission check can intervene. */
+export async function readCandidateMetadata(
+  path: string,
+  scope: CandidateMetadataScope,
+  limit: number,
+): Promise<Uint8Array> {
+  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+  try {
+    const stat = await handle.stat();
+    if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size > limit)
+      throw Error("CANDIDATE_METADATA_FILE_LIMIT");
+    // Conservative attribution includes strings, small JSON nodes, duplicate
+    // checked representations, digest serialization and installed indexes.
+    scope.charge((stat.size + 1) * 32);
+    const bytes = Buffer.alloc(stat.size + 1);
+    let offset = 0;
+    while (offset < bytes.length) {
+      const read = await handle.read(bytes, offset, bytes.length - offset, offset);
+      if (!read.bytesRead) break;
+      offset += read.bytesRead;
+    }
+    if (offset !== stat.size) throw Error("CANDIDATE_METADATA_CHANGED");
+    return bytes.subarray(0, offset);
+  } finally {
+    await handle.close();
+  }
+}

src/execution-world/ports.ts

@@ -29,6 +29,8 @@ export interface PackageInvocation {
   /** Capability arguments/receiver use the versioned managed-reference codec. */
   input: unknown;
   receiver: unknown | null;
+  /** Host-owned remaining root resources, separate from authored input. */
+  budget?: { root: string; depth: number; calls: number; deadlineMs: number };
 }
 export type InvocationReply =
   | { abi: typeof transactionPortAbi; invocation: string; status: "returned"; output: unknown }

test/candidate-metadata.test.ts

@@ -0,0 +1,50 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, writeFile, symlink, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { CandidateMetadataBudget, readCandidateMetadata } from "../src/execution-world/metadata.js";
+
+test("concurrent candidate reservations enforce aggregate and owner limits and release exactly once", () => {
+  const budget = new CandidateMetadataBudget({ bytes: 1000, ownerBytes: 600, candidates: 3 });
+  const first = budget.acquire("a"),
+    second = budget.acquire("a"),
+    third = budget.acquire("b");
+  assert.throws(() => budget.acquire("c"), /CANDIDATE_METADATA_LIMIT/);
+  first.charge(400);
+  second.charge(200);
+  third.charge(400);
+  assert.throws(() => second.charge(1), /CANDIDATE_METADATA_LIMIT/);
+  assert.throws(() => third.charge(1), /CANDIDATE_METADATA_LIMIT/);
+  assert.equal(budget.snapshot().bytes, 1000);
+  second.release();
+  second.release();
+  assert.equal(budget.snapshot().bytes, 800);
+  assert.throws(() => second.charge(1), /CANDIDATE_METADATA_CLOSED/);
+  first.release();
+  third.release();
+  assert.deepEqual(budget.snapshot(), { bytes: 0, candidates: 0, owners: 0, limits: budget.limits });
+});
+
+test("metadata files reserve parse/index space before allocation and refuse oversized or linked inputs", async () => {
+  const root = await mkdtemp(join(tmpdir(), "qx-metadata-budget-"));
+  const budget = new CandidateMetadataBudget({ bytes: 512, ownerBytes: 512, candidates: 2 });
+  const scope = budget.acquire("workspace");
+  try {
+    const file = join(root, "metadata.json");
+    await writeFile(file, '{"ok":true}');
+    assert.equal(new TextDecoder().decode(await readCandidateMetadata(file, scope, 100)), '{"ok":true}');
+    assert.equal(scope.bytes, 12 * 32);
+    await assert.rejects(readCandidateMetadata(file, scope, 100), /CANDIDATE_METADATA_LIMIT/);
+    assert.equal(scope.bytes, 12 * 32);
+    await assert.rejects(readCandidateMetadata(file, scope, 5), /CANDIDATE_METADATA_FILE_LIMIT/);
+    await symlink(file, join(root, "linked"));
+    await assert.rejects(readCandidateMetadata(join(root, "linked"), scope, 100), { code: "ELOOP" });
+    await assert.rejects(readCandidateMetadata(root, scope, 100), /CANDIDATE_METADATA_FILE_LIMIT/);
+    assert.equal(scope.bytes, 12 * 32);
+  } finally {
+    scope.release();
+    await rm(root, { recursive: true, force: true });
+  }
+  assert.equal(budget.snapshot().bytes, 0);
+});