Pool host-approved restricted code with scoped and fair graph execution

src/execution-world/candidate.ts

@@ -11,8 +11,12 @@ import {
 } from "./metadata.js";
 const rootPattern = /^\/nix\/store\/[a-z0-9]{32}-[^/\s]+$/;
 async function readJson(root: string, file: string, scope: CandidateMetadataScope): Promise<any> {
-  const bytes = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024);
-  return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
+  const input = await readCandidateMetadata(join(root, file), scope, 32 * 1024 * 1024);
+  try {
+    return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(input.bytes));
+  } finally {
+    input.releaseInput();
+  }
 }
 /** Load the checked Nix output, preserving its original compilation and provenance.
  * The selected toolchain validates generated code during the build. This loader

src/execution-world/files.ts

@@ -13,7 +13,7 @@ import {
   type CheckedWorld,
 } from "./artifacts.js";
 import { type ExecutionPlan, ExecutionPlanError } from "./plan.js";
-import type { CandidateMetadataScope } from "./metadata.js";
+import { readCandidateMetadata, type CandidateMetadataScope } from "./metadata.js";
 
 const maxFileBytes = 256 * 1024 * 1024;
 const metadataLimit = 16 * 1024 * 1024;
@@ -22,10 +22,19 @@ const fail = (where: string, message: string): never => {
 };
 /** Roots must be trusted immutable output directories. Refuse symlinks, including
  * interior directory links. Hash verification is still required before serving bytes. */
-export async function artifactFileReader(
-  roots: Readonly<Record<string, string>>,
-  metadata?: CandidateMetadataScope,
-): Promise<ArtifactReader> {
+async function artifactFile(root: string, relative: string): Promise<string> {
+  artifactPath(relative);
+  const components = relative.split("/");
+  let resolved = root;
+  for (let index = 0; index < components.length; index++) {
+    resolved = path.join(resolved, components[index]!);
+    const stat = await lstat(resolved);
+    if (stat.isSymbolicLink() || (index < components.length - 1 ? !stat.isDirectory() : !stat.isFile()))
+      fail(relative, "Only real directories and regular files are allowed");
+  }
+  return resolved;
+}
+export async function artifactFileReader(roots: Readonly<Record<string, string>>): Promise<ArtifactReader> {
   const directories = new Map<string, string>();
   for (const [revision, root] of Object.entries(roots)) {
     if (!path.isAbsolute(root) || path.normalize(root) !== root || (await realpath(root)) !== root)
@@ -37,21 +46,12 @@ export async function artifactFileReader(
   return async (revision, relative, limit) => {
     const root = directories.get(revision);
     if (!root) fail(revision, "Unknown package output");
-    artifactPath(relative);
     if (!Number.isSafeInteger(limit) || limit < 0 || limit > maxFileBytes) fail(relative, "Invalid file byte limit");
-    const components = relative.split("/");
-    let resolved = root!;
-    for (let index = 0; index < components.length; index++) {
-      resolved = path.join(resolved, components[index]!);
-      const stat = await lstat(resolved);
-      if (stat.isSymbolicLink() || (index < components.length - 1 ? !stat.isDirectory() : !stat.isFile()))
-        fail(relative, "Only real directories and regular files are allowed");
-    }
+    const resolved = await artifactFile(root!, relative);
     const handle = await open(resolved, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
     try {
       const stat = await handle.stat();
       if (!stat.isFile() || stat.size > limit) fail(relative, "File exceeds its declared bound");
-      if (relative === "share/quixos/package-artifacts.json") metadata?.charge((stat.size + 1) * 32);
       // A fixed allocation and one extra byte detect growth without readFile's unbounded allocation.
       const buffer = Buffer.alloc(stat.size + 1);
       let offset = 0;
@@ -79,14 +79,22 @@ export async function loadArtifactWorld(
   roots: Readonly<Record<string, string>>,
   metadata?: CandidateMetadataScope,
 ): Promise<LoadedArtifactWorld> {
-  const read = await artifactFileReader(roots, metadata),
+  const read = await artifactFileReader(roots),
     manifests: PackageArtifacts[] = [];
   const expected = plan.definition.packages.map((p) => p.revision).sort();
   if (JSON.stringify(Object.keys(roots).sort()) !== JSON.stringify(expected))
     fail("world", "Output roots must exactly match the installed plan");
   for (const revision of expected) {
-    const bytes = await read(revision, "share/quixos/package-artifacts.json", metadataLimit);
-    const manifest = parsePackageArtifacts(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)));
+    const file = "share/quixos/package-artifacts.json";
+    const input = metadata
+      ? await readCandidateMetadata(await artifactFile(roots[revision]!, file), metadata, metadataLimit)
+      : { bytes: await read(revision, file, metadataLimit), releaseInput: () => {} };
+    let manifest: PackageArtifacts;
+    try {
+      manifest = parsePackageArtifacts(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(input.bytes)));
+    } finally {
+      input.releaseInput();
+    }
     if (manifest.packageRevisionId !== revision) fail(revision, "Output belongs to another package revision");
     manifests.push(manifest);
   }

src/execution-world/metadata.ts

@@ -24,28 +24,44 @@ export class CandidateMetadataBudget {
     this.scopes++;
     let bytes = 0,
       closed = false;
+    const charge = (amount: number) => {
+      if (closed) throw Error("CANDIDATE_METADATA_CLOSED");
+      if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE");
+      const owned = this.owners.get(owner) ?? 0;
+      if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes)
+        throw Error("CANDIDATE_METADATA_LIMIT");
+      bytes += amount;
+      this.bytes += amount;
+      this.owners.set(owner, owned + amount);
+    };
+    const subtract = (amount: number) => {
+      bytes -= amount;
+      this.bytes -= amount;
+      const remaining = (this.owners.get(owner) ?? 0) - amount;
+      if (remaining) this.owners.set(owner, remaining);
+      else this.owners.delete(owner);
+    };
     return {
       get bytes() {
         return bytes;
       },
-      charge: (amount: number) => {
-        if (closed) throw Error("CANDIDATE_METADATA_CLOSED");
-        if (!Number.isSafeInteger(amount) || amount < 0) throw Error("INVALID_CANDIDATE_METADATA_CHARGE");
-        const owned = this.owners.get(owner) ?? 0;
-        if (this.bytes + amount > this.limits.bytes || owned + amount > this.limits.ownerBytes)
-          throw Error("CANDIDATE_METADATA_LIMIT");
-        bytes += amount;
-        this.bytes += amount;
-        this.owners.set(owner, owned + amount);
+      charge,
+      /** Opaque, idempotent transient reservation; callers cannot subtract an
+       * arbitrary amount or release another candidate's charges. */
+      reserve: (amount: number) => {
+        charge(amount);
+        let released = false;
+        return () => {
+          if (closed || released) return;
+          released = true;
+          subtract(amount);
+        };
       },
       release: () => {
         if (closed) return;
         closed = true;
-        this.bytes -= bytes;
+        subtract(bytes);
         this.scopes--;
-        const remaining = (this.owners.get(owner) ?? 0) - bytes;
-        if (remaining) this.owners.set(owner, remaining);
-        else this.owners.delete(owner);
       },
     };
   }
@@ -53,21 +69,50 @@ export class CandidateMetadataBudget {
 export type CandidateMetadataScope = ReturnType<CandidateMetadataBudget["acquire"]>;
 export const candidateMetadataBudget = new CandidateMetadataBudget();
 
+// Estimate JSON allocation without constructing its tree. Object/array headers,
+// value slots and strings get explicit weights, with a multiplier for checked
+// indexes/derived representations. Indentation must not multiply tree charges.
+function jsonAllocation(bytes: Uint8Array): number {
+  let estimate = 0;
+  for (let i = 0; i < bytes.length; i++) {
+    const c = bytes[i]!;
+    if (c === 123) estimate += 64;
+    else if (c === 91) estimate += 32;
+    else if (c === 34) {
+      const start = ++i;
+      while (i < bytes.length && bytes[i] !== 34) {
+        if (bytes[i] === 92) i++;
+        i++;
+      }
+      estimate += 32 + 2 * (i - start);
+    } else if (c === 45 || (c >= 48 && c <= 57) || c === 116 || c === 102 || c === 110) {
+      estimate += 16;
+      while (i + 1 < bytes.length) {
+        const next = bytes[i + 1]!;
+        if (next === 9 || next === 10 || next === 13 || next === 32 || next === 44 || next === 93 || next === 125)
+          break;
+        i++;
+      }
+    }
+  }
+  return estimate * 2;
+}
+
 /** Immutable regular files only. Fixed allocation plus an extra byte catches
  * growth; readFile() would allocate before the admission check can intervene. */
 export async function readCandidateMetadata(
   path: string,
   scope: CandidateMetadataScope,
   limit: number,
-): Promise<Uint8Array> {
+): Promise<{ bytes: Uint8Array; releaseInput: () => void }> {
   const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+  let releaseInput: (() => void) | undefined;
   try {
     const stat = await handle.stat();
     if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size > limit)
       throw Error("CANDIDATE_METADATA_FILE_LIMIT");
-    // Conservative attribution includes strings, small JSON nodes, duplicate
-    // checked representations, digest serialization and installed indexes.
-    scope.charge((stat.size + 1) * 32);
+    // Reserve the input buffer and UTF-16 decoding before allocating either.
+    releaseInput = scope.reserve((stat.size + 1) * 3);
     const bytes = Buffer.alloc(stat.size + 1);
     let offset = 0;
     while (offset < bytes.length) {
@@ -76,7 +121,13 @@ export async function readCandidateMetadata(
       offset += read.bytesRead;
     }
     if (offset !== stat.size) throw Error("CANDIDATE_METADATA_CHANGED");
-    return bytes.subarray(0, offset);
+    const content = bytes.subarray(0, offset);
+    // No JSON.parse/decoded tree exists until its shape-based charge succeeds.
+    scope.charge(jsonAllocation(content));
+    return { bytes: content, releaseInput };
+  } catch (error) {
+    releaseInput?.();
+    throw error;
   } finally {
     await handle.close();
   }

src/execution-world/placement.ts

@@ -137,8 +137,9 @@ export function inspectJavaScriptSurface(source: string, checkedSDKImports: read
 }
 
 /** Derived metadata only. Browser/server portability and available targets say
- * nothing about sandbox eligibility. No existing package artifact has a
- * qualified restricted runtime, so it conservatively requires its VM. */
+ * nothing about sandbox eligibility. Build reports have no platform approval
+ * catalog, so they conservatively require VM placement. A host can override
+ * placement only for independently reviewed, exact restricted-code approvals. */
 export async function reportExecutionPlacement(
   plan: ExecutionPlan,
   installation: ExecutionInstallation,

test/candidate-metadata.test.ts

@@ -28,23 +28,74 @@ test("concurrent candidate reservations enforce aggregate and owner limits and r
 
 test("metadata files reserve parse/index space before allocation and refuse oversized or linked inputs", async () => {
   const root = await mkdtemp(join(tmpdir(), "qx-metadata-budget-"));
-  const budget = new CandidateMetadataBudget({ bytes: 512, ownerBytes: 512, candidates: 2 });
+  const budget = new CandidateMetadataBudget({ bytes: 450, ownerBytes: 450, candidates: 2 });
   const scope = budget.acquire("workspace");
   try {
     const file = join(root, "metadata.json");
     await writeFile(file, '{"ok":true}');
-    assert.equal(new TextDecoder().decode(await readCandidateMetadata(file, scope, 100)), '{"ok":true}');
-    assert.equal(scope.bytes, 12 * 32);
+    const input = await readCandidateMetadata(file, scope, 100);
+    assert.equal(new TextDecoder().decode(input.bytes), '{"ok":true}');
+    const withInput = scope.bytes;
+    input.releaseInput();
+    input.releaseInput();
+    const first = scope.bytes;
+    assert.equal(withInput - first, (Buffer.byteLength('{"ok":true}') + 1) * 3);
+    assert.ok(first > 12 * 3, "Decoded tree and derived metadata are reserved before parsing");
     await assert.rejects(readCandidateMetadata(file, scope, 100), /CANDIDATE_METADATA_LIMIT/);
-    assert.equal(scope.bytes, 12 * 32);
+    assert.equal(scope.bytes, first, "Rejected tree allocation releases its transient input buffer");
     await assert.rejects(readCandidateMetadata(file, scope, 5), /CANDIDATE_METADATA_FILE_LIMIT/);
     await symlink(file, join(root, "linked"));
     await assert.rejects(readCandidateMetadata(join(root, "linked"), scope, 100), { code: "ELOOP" });
     await assert.rejects(readCandidateMetadata(root, scope, 100), /CANDIDATE_METADATA_FILE_LIMIT/);
-    assert.equal(scope.bytes, 12 * 32);
+    assert.equal(scope.bytes, first);
   } finally {
     scope.release();
     await rm(root, { recursive: true, force: true });
   }
   assert.equal(budget.snapshot().bytes, 0);
 });
+
+test("JSON structure accounting is independent of indentation and covers strings, values and dense nodes", async () => {
+  const root = await mkdtemp(join(tmpdir(), "qx-metadata-shape-"));
+  const budget = new CandidateMetadataBudget();
+  try {
+    for (const value of [{}, [null, true, false, 1, -3.5e8], { 'a"b': '雪\\"', nodes: [{}, {}, {}] }]) {
+      const measured = [];
+      for (const spaces of [0, 4]) {
+        const file = join(root, "metadata.json"),
+          text = JSON.stringify(value, null, spaces);
+        await writeFile(file, text);
+        const scope = budget.acquire("test");
+        try {
+          const input = await readCandidateMetadata(file, scope, 10000);
+          assert.deepEqual(JSON.parse(new TextDecoder().decode(input.bytes)), value);
+          input.releaseInput();
+          measured.push(scope.bytes);
+        } finally {
+          scope.release();
+        }
+      }
+      assert.equal(measured[0], measured[1], "Whitespace changes input storage, not the allocated JSON shape");
+      assert.ok(measured[0] >= 128);
+    }
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+  assert.equal(budget.snapshot().bytes, 0);
+});
+
+test("transient leases release exactly once, including after their parent closes", () => {
+  const budget = new CandidateMetadataBudget({ bytes: 1000, ownerBytes: 600, candidates: 3 });
+  const scope = budget.acquire("a");
+  scope.charge(200);
+  const release = scope.reserve(300);
+  assert.throws(() => scope.reserve(101), /CANDIDATE_METADATA_LIMIT/);
+  release();
+  release();
+  assert.equal(scope.bytes, 200);
+  const afterClose = scope.reserve(100);
+  scope.release();
+  afterClose();
+  assert.equal(scope.bytes, 0);
+  assert.equal(budget.snapshot().bytes, 0);
+});