diff --git a/flake.nix b/flake.nix index 54daab3..80677c9 100644 --- a/flake.nix +++ b/flake.nix @@ -81,11 +81,13 @@ install -Dm644 nix/checked-package.nix "$out/share/checked-package.nix" install -Dm644 nix/checked-artifacts.nix "$out/share/checked-artifacts.nix" install -Dm644 nix/package-context.nix "$out/share/package-context.nix" + install -Dm644 nix/lock-source.nix "$out/share/lock-source.nix" install -Dm644 nix/package-context.mjs "$out/share/package-context.mjs" install -Dm644 nix/check-package.mjs "$out/share/check-package.mjs" install -Dm644 nix/check-execution.mjs "$out/share/check-execution.mjs" substitute nix/checked-candidate.nix "$out/share/checked-candidate.nix" \ - --replace-fail '@nixpkgs@' '${pkgs.path}' + --replace-fail '@nixpkgs@' '${pkgs.path}' \ + --replace-fail '@nixpkgsNarHash@' '${nixpkgs.narHash}' cp --reflink=auto --recursive grammar "$out/grammar" cp --reflink=auto --recursive proto "$out/proto" cp --reflink=auto --recursive dist "$out/dist" diff --git a/nix/checked-candidate.nix b/nix/checked-candidate.nix index 2cebdd7..d657ca0 100644 --- a/nix/checked-candidate.nix +++ b/nix/checked-candidate.nix @@ -9,7 +9,11 @@ system ? builtins.currentSystem, }: let - pkgs = import (/. + "@nixpkgs@") { inherit system; }; + pkgs = import (/. + "@nixpkgs@") { + inherit system; + config = { }; + overlays = [ ]; + }; protocol = builtins.storePath generator; sourceKey = source: "${source.kind}:${source.repository}@${source.commit}"; fetch = @@ -26,8 +30,9 @@ let source: let directory = fetch source; + lockSource = import ./lock-source.nix { source = directory; }; lockFile = pkgs.runCommand "qx-source-lock.json" { } '' - ${protocol}/bin/quixos-lock-check ${directory}/quixos.lock > "$out" + ${protocol}/bin/quixos-lock-check ${lockSource}/quixos.lock > "$out" ''; lock = builtins.fromJSON (builtins.readFile lockFile); in @@ -124,15 +129,40 @@ let }) (builtins.filter (entry: entry.kind != "workspace") (resourceClosure true node)); } ); - compile = + # Each resource is checked once per exact source/closure/frontend/toolchain. + # Composition imports these compiler outputs; authoring receipts are never + # build inputs. Missing entries in ordinary CLI use take the full check path. + compiledNodes = builtins.mapAttrs (_: node: compileNode node) nodeTable; + compile = node: compiledNodes.${sourceKey node}; + checkedResources = node: - pkgs.runCommand "qx-${node.kind}-contract" { } '' + pkgs.writeText "qx-checked-resources.json" ( + builtins.toJSON ( + map + (dependency: { + inherit (dependency) kind repository commit; + artifact = compile dependency; + }) + ( + builtins.filter ( + dependency: + dependency.kind != "workspace" + && dependency.kind != "build" + && sourceKey dependency != sourceKey node + ) (resourceClosure true node) + ) + ) + ); + compileNode = + node: + pkgs.runCommand "qx-${node.kind}-contract" { QUIXOS_TRACE_PROFILE = "1"; } '' mkdir -p "$out" ${ if node.kind == "workspace" then '' ${protocol}/bin/quixos-workspace-compile --root ${node.directory} \ - ${queryGeneratorFlags node} \ + ${queryGeneratorFlags root} \ + --checked-resources ${checkedResources node} \ --source-root-commit ${pkgs.lib.escapeShellArg node.commit} \ --checkout-root "$TMPDIR/checkouts" --snapshot-map ${snapshots node} \ --graph-out "$out/graph.json" --schemas-out "$out/package-schemas.json" \ @@ -144,7 +174,8 @@ let else '' ${protocol}/bin/quixos-resource-compile --root ${node.directory} \ - ${queryGeneratorFlags node} \ + ${queryGeneratorFlags root} \ + --checked-resources ${checkedResources node} \ --kind ${node.kind} --repository ${pkgs.lib.escapeShellArg node.repository} \ --commit ${pkgs.lib.escapeShellArg node.commit} \ --checkout-root "$TMPDIR/checkouts" --snapshot-map ${snapshots node} --snapshot-only true \ @@ -153,6 +184,18 @@ let } ''; contract = compile root; + # Parse shared workspace outputs once. Repeating fromJSON in every package + # makes composition quadratic in package count and in the serialized world. + workspacePackages = builtins.listToAttrs ( + map + (pkg: { + name = sourceKey (pkg.source // { kind = "package"; }); + value = pkg; + }) + (builtins.fromJSON (builtins.readFile "${contract}/execution-plan.json")) + .definition.bindingSchema.packages + ); + workspaceContexts = builtins.fromJSON (builtins.readFile "${contract}/build-contexts.json"); buildDependencies = node: builtins.listToAttrs ( @@ -218,17 +261,19 @@ let if execution && kind == "workspace" then { revision = - pkgs.lib.findFirst - (pkg: pkg.source.repository == node.repository && pkg.source.commit == node.commit) - (throw "Source package missing from workspace compilation: ${node.repository}") - (builtins.fromJSON (builtins.readFile "${contract}/execution-plan.json")) - .definition.bindingSchema.packages; + workspacePackages.${sourceKey node} + or (throw "Source package missing from workspace compilation: ${node.repository}"); } else builtins.fromJSON (builtins.readFile "${compiled}/candidate.json"); buildContext = import ./package-context.nix { inherit pkgs protocol; contexts = if kind == "workspace" then "${contract}/build-contexts.json" else null; + contextData = + if execution && kind == "workspace" then + workspaceContexts.${candidate.revision.revisionId} + else + null; plan = "${compiled}/bindings.json"; packageRevisionId = candidate.revision.revisionId; }; @@ -254,6 +299,11 @@ let typescript = runtimePackages.camino-replica-core.dependencies; packageRevisionId = candidate.revision.revisionId; dependencies = buildDependencies node; + buildEnvironment = { + inherit pkgs system; + nixpkgsSource = toString pkgs.path; + narHash = "@nixpkgsNarHash@"; + }; } // pkgs.lib.optionalAttrs (componentEntries != [ ]) { react = { @@ -310,7 +360,7 @@ in pkgs.runCommand (if contractOnly then "qx-contract" else "qx-checked-candidate") { } '' mkdir -p "$out" cp ${ - pkgs.writeText "qx-candidate-source.json" (builtins.toJSON { inherit repository commit kind; }) + builtins.toFile "qx-candidate-source.json" (builtins.toJSON { inherit repository commit kind; }) } "$out/source.json" cp ${contract}/candidate.json "$out/candidate.json" cp ${contract}/graph.json "$out/graph.json" diff --git a/nix/lock-source.nix b/nix/lock-source.nix new file mode 100644 index 0000000..bceef63 --- /dev/null +++ b/nix/lock-source.nix @@ -0,0 +1,16 @@ +# Lock parsing only reads this one file when no import token can occur. A +# conservative substring probe also sends comments/URLs containing "import" +# through the complete-source path. Fragment names may have any extension. +{ source }: +let + regular = ((builtins.readDir source)."quixos.lock" or null) == "regular"; + text = if regular then builtins.readFile (source + "/quixos.lock") else "import"; +in +if builtins.replaceStrings [ "import" ] [ "" ] text != text then + source +else + builtins.path { + path = source; + name = "qx-lock-source"; + filter = filename: _: filename == toString source || filename == "${source}/quixos.lock"; + } diff --git a/nix/lock-source.test.nix b/nix/lock-source.test.nix new file mode 100644 index 0000000..c1669f4 --- /dev/null +++ b/nix/lock-source.test.nix @@ -0,0 +1,42 @@ +{ pkgs, protocol }: +let + base = ''quixos-lock version 1 { quixos source { repository "https://example.test/quixos.git"; commit "1111111111111111111111111111111111111111"; } }''; + fixture = + extra: lock: + pkgs.runCommand "qx-lock-source-fixture" { } '' + mkdir -p "$out/nested" + printf '%s' ${pkgs.lib.escapeShellArg lock} > "$out/quixos.lock" + printf '%s' ${pkgs.lib.escapeShellArg extra} > "$out/implementation.ts" + printf '%s' 'quixos-lock fragment version 1 {}' > "$out/nested/dependencies.txt" + ''; + first = fixture "first implementation" base; + edited = fixture "edited implementation" base; + changed = fixture "first implementation" ( + builtins.replaceStrings + [ "1111111111111111111111111111111111111111" ] + [ "2222222222222222222222222222222222222222" ] + base + ); + fragments = fixture "first implementation" ( + builtins.replaceStrings [ "quixos source" ] [ ''import "nested/dependencies.txt"; quixos source'' ] + base + ); + conservative = fixture "first implementation" (base + "\n// import in a comment\n"); + slice = source: import ./lock-source.nix { inherit source; }; + parse = + source: + pkgs.runCommand "qx-source-lock.json" { } '' + ${protocol}/bin/quixos-lock-check ${slice source}/quixos.lock > "$out" + ''; +in +assert slice first == slice edited; +assert (parse first).drvPath == (parse edited).drvPath; +assert slice first != slice changed; +pkgs.runCommand "qx-lock-source-check" { } '' + ${protocol}/bin/quixos-lock-check ${slice first}/quixos.lock > first.json + ${protocol}/bin/quixos-lock-check ${slice edited}/quixos.lock > edited.json + cmp first.json edited.json + ${protocol}/bin/quixos-lock-check ${slice conservative}/quixos.lock > comment.json + cmp first.json comment.json + ${protocol}/bin/quixos-lock-check ${slice fragments}/quixos.lock > "$out" +'' diff --git a/nix/package-context.nix b/nix/package-context.nix index 7317379..ef20489 100644 --- a/nix/package-context.nix +++ b/nix/package-context.nix @@ -5,6 +5,7 @@ protocol, plan ? null, contexts ? null, + contextData ? null, packageRevisionId, }: let @@ -14,7 +15,9 @@ let ${pkgs.lib.escapeShellArg packageRevisionId} > "$out" ''; context = - if contexts == null then + if contextData != null then + contextData + else if contexts == null then builtins.fromJSON (builtins.readFile projected) else (builtins.fromJSON (builtins.readFile contexts)).${packageRevisionId}; diff --git a/src/capability-language/assembly.ts b/src/capability-language/assembly.ts index 332bca0..6236e1d 100644 --- a/src/capability-language/assembly.ts +++ b/src/capability-language/assembly.ts @@ -1,5 +1,6 @@ import { readFile, realpath } from "node:fs/promises"; import path from "node:path"; +import { profile } from "./profile.js"; import { loadQxSources, resolveQxSources } from "./source-loader.js"; import { compileQuery, compileRepositoryQuery } from "../query/compile.js"; import type { QueryDeclaration } from "../query/types.js"; @@ -43,6 +44,15 @@ export type CapabilityRepositoryResolver = ( kind: LockedResource["kind"], ) => Promise; +/** Already checked outputs of this compiler, composed by the Nix build DAG. + * Source capture/check reports never grant this authority. */ +export type CheckedResourceResolver = ( + kind: LockedResource["kind"], + source: GitSource, + lock: QuixosRepositoryLock, + dependencies: ReadonlyMap, +) => Promise; + /** An explicitly selected build frontend supplies ordinary query sources. The * assembler never chooses a frontend by inspecting an interface/contract ID. * Generated sources go through the same query and imported-fragment checks. */ @@ -221,7 +231,12 @@ const createResourceGraphResolver = ( quixosCommit: string, resolveResource: CapabilityRepositoryResolver, generateQueries?: PackageQueryGenerator, + checkedResource?: CheckedResourceResolver, ) => { + const graphStarted = performance.now(); + let checkedResources = 0, + reusedResources = 0, + checkedQueries = 0; const resolved = new Map>(); const active: string[] = []; @@ -275,6 +290,30 @@ const createResourceGraphResolver = ( }; } const dependencyClosure = resourceClosure(directPairs.map(([, node]) => node)); + const dependencies = new Map( + directPairs.map(([dependency, node]) => [bindingKey(dependency.kind, dependency.binding), node]), + ); + const checked = await checkedResource?.(locked.kind, locked.source, lockResult.lock, dependencies); + if (checked) { + if ( + checked.kind !== locked.kind || + checked.revision.source.repository !== locked.source.repository || + checked.revision.source.commit !== locked.source.commit + ) + throw new Error("Checked resource identity mismatch"); + assertImportsMatchLock("Checked resource", checked.imports, lockResult.lock.resources); + reusedResources++; + return { + key, + kind: locked.kind, + source: locked.source, + directory: snapshot.directory, + lock: lockResult.lock, + resource: structuredClone(checked), + dependencies, + }; + } + const checkStarted = performance.now(); const environment = environmentFor(directPairs, dependencyClosure); const manifestName = locked.kind === "interface" ? "interface.qx" : "package.qx"; const manifestPath = path.join(snapshot.directory, manifestName); @@ -399,6 +438,15 @@ const createResourceGraphResolver = ( compiled.resource.revision.migrationCatalog = catalog; } } + checkedResources++; + const queries = + compiled.resource.kind === "package" ? (compiled.resource.revision.checkedQueries?.length ?? 0) : 0; + checkedQueries += queries; + profile("resource-check", checkStarted, { + kind: locked.kind, + resource: compiled.resource.revision.displayName, + queries, + }); return { key, kind: locked.kind, @@ -420,7 +468,11 @@ const createResourceGraphResolver = ( return { visit, - nodes: async () => await Promise.all(resolved.values()), + nodes: async () => { + const nodes = await Promise.all(resolved.values()); + profile("resource-graph", graphStarted, { checkedResources, reusedResources, checkedQueries }); + return nodes; + }, }; }; @@ -430,6 +482,7 @@ export const compileCapabilityResourceRepository = async (options: { source: GitSource; resolveResource: CapabilityRepositoryResolver; generateQueries?: PackageQueryGenerator; + checkedResource?: CheckedResourceResolver; }): Promise => { const lockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock")); if (!lockResult.ok) { @@ -439,6 +492,7 @@ export const compileCapabilityResourceRepository = async (options: { lockResult.lock.quixos.commit, options.resolveResource, options.generateQueries, + options.checkedResource, ); const root = await resolver.visit( { @@ -466,6 +520,7 @@ export const compileWorkspaceRepository = async (options: { /** An editor's proposed source snapshot; locks and dependency revisions remain exact. */ readSource?: (name: string) => Promise; generateQueries?: PackageQueryGenerator; + checkedResource?: CheckedResourceResolver; }): Promise => { const rootLockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock")); if (!rootLockResult.ok) { @@ -476,6 +531,7 @@ export const compileWorkspaceRepository = async (options: { rootLock.quixos.commit, options.resolveResource, options.generateQueries, + options.checkedResource, ); const directPairs: Array<[LockedResource, ResolvedCapabilityResource]> = []; diff --git a/src/capability-language/authoring-check.ts b/src/capability-language/authoring-check.ts index 5f65355..b378075 100644 --- a/src/capability-language/authoring-check.ts +++ b/src/capability-language/authoring-check.ts @@ -64,7 +64,7 @@ export async function checkAuthoring( // Repository-scoped agents can check separate immutable candidates in parallel. const capture = promisify(callback)("quixos-qx", ["converge", context.workbench, directory], { maxBuffer: 4 * 1024 * 1024, - env: { ...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_TRACE_CAPTURE: "1" }, + env: { ...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_TRACE_CAPTURE: "1", QUIXOS_TRACE_PROFILE: "1" }, }); console.error(`Capture details: ${path.join(output, "capture.log")}`); capture.child.stderr?.on("data", (chunk) => appendFileSync(path.join(output, "capture.log"), chunk)); @@ -94,6 +94,9 @@ export async function checkAuthoring( resource.kind, path.join(output, "nix.log"), options.contractOnly, + (phase, ms) => { + timings[phase] = ms; + }, ); } finally { timings.immutableCheckMs = Math.round(performance.now() - buildStarted); diff --git a/src/capability-language/authoring-converge.ts b/src/capability-language/authoring-converge.ts index c410152..cba7d94 100644 --- a/src/capability-language/authoring-converge.ts +++ b/src/capability-language/authoring-converge.ts @@ -5,6 +5,7 @@ import { promisify } from "node:util"; import { randomUUID } from "node:crypto"; import { authoringContext } from "./authoring-context.js"; import { snapshotCommit } from "./checked-build.js"; +import { profile } from "./profile.js"; import { loadQuixosLock, parseQuixosLockDocument, @@ -14,6 +15,42 @@ import { } from "../resource-lock/index.js"; const execFile = promisify(callback); +// Bound independent repository operations. One visit owns each repository; +// all visits finish before the final concurrent-edit validation pass. +const concurrency = 4; +async function mapConcurrent(values: readonly T[], action: (value: T) => Promise): Promise { + const results = new Array(values.length); + let next = 0; + await Promise.all( + Array.from({ length: Math.min(concurrency, values.length) }, async () => { + while (next < values.length) { + const index = next++; + results[index] = await action(values[index]); + } + }), + ); + return results; +} +const limiter = () => { + let running = 0; + const waiting: Array<() => void> = []; + return async (action: () => Promise): Promise => { + await new Promise((resolve) => { + if (running < concurrency) { + running++; + resolve(); + } else waiting.push(resolve); + }); + try { + return await action(); + } finally { + const next = waiting.shift(); + if (next) next(); + else running--; + } + }; +}; + const command = async (cwd: string, executable: string, args: string[]) => { const start = performance.now(); // Do not log arguments: transports may contain credentials. Source identities @@ -44,13 +81,14 @@ export type AuthoringBlocker = { * worklist grants typechecking, semantic review or activation approval. * Caller serializes coordinators; package authors may still be editing. */ export async function convergeAuthoring(start: string, target = "root") { + const started = performance.now(); const context = await authoringContext(start); const graphFile = path.join(context.workbench, ".quixos/resource-graph.json"); const graphBefore = await readFile(graphFile, "utf8"); const blockers: AuthoringBlocker[] = []; const nodes = new Map(); const selected = new Map(); - for (const entry of context.resources) { + const inventory = await mapConcurrent(context.resources, async (entry) => { const root = path.join(context.workbench, entry.directory); let repository = entry.source?.repository; try { @@ -64,6 +102,9 @@ export async function convergeAuthoring(start: string, target = "root") { blockers.push({ directory: entry.directory, phase: "source", message: String(error).slice(0, 2000) }); if (!repository) throw error; // The root has no separate registered source. } + return { ...entry, repository }; + }); + for (const { repository, ...entry } of inventory) { const key = identity(entry.kind, repository); if (selected.has(key)) throw new Error(`More than one editable checkout for ${repository}`); selected.set(key, entry.directory); @@ -73,7 +114,7 @@ export async function convergeAuthoring(start: string, target = "root") { dependencies: [], }); } - for (const node of nodes.values()) { + await mapConcurrent([...nodes.values()], async (node) => { try { const lock = await loadQuixosLock(path.join(context.workbench, node.directory, "quixos.lock")); if (!lock.ok) throw new Error(lock.diagnostics.map((d) => `${d.fileName}: ${d.message}`).join("\n")); @@ -88,97 +129,122 @@ export async function convergeAuthoring(start: string, target = "root") { } catch (error) { blockers.push({ directory: node.directory, phase: "resolution", message: String(error) }); } - } - const complete = new Map(), - active = new Set(); - const visited = new Set(); - if (!nodes.has(target)) throw new Error(`Not a registered repository: ${target}`); - const visit = async (directory: string): Promise => { - visited.add(directory); - if (complete.has(directory)) return true; - if (blockers.some((entry) => entry.directory === directory)) return false; - if (active.has(directory)) { + }); + // Detect cycles before starting concurrent futures: two roots awaiting each + // other must fail clearly rather than deadlocking on their cached promises. + const scanned = new Set(); + const detectCycles = (directory: string, parents: string[] = []) => { + if (parents.includes(directory)) { blockers.push({ directory, phase: "dependency", - message: `Source dependency cycle: ${[...active, directory].join(" -> ")}`, + message: `Source dependency cycle: ${[...parents, directory].join(" -> ")}`, }); - return false; + return; } - active.add(directory); - const node = nodes.get(directory)!; - for (const dependency of node.dependencies) - if (!(await visit(dependency))) { - blockers.push({ directory, phase: "dependency", message: `Waiting for ${dependency}` }); - active.delete(directory); + if (scanned.has(directory)) return; + for (const dependency of nodes.get(directory)!.dependencies) detectCycles(dependency, [...parents, directory]); + scanned.add(directory); + }; + for (const directory of nodes.keys()) detectCycles(directory); + const complete = new Map(); + const pending = new Map>(); + const capture = limiter(); + const visited = new Set(); + if (!nodes.has(target)) throw new Error(`Not a registered repository: ${target}`); + const visit = (directory: string): Promise => { + visited.add(directory); + const existing = pending.get(directory); + if (existing) return existing; + const result = (async () => { + if (blockers.some((entry) => entry.directory === directory)) return false; + const node = nodes.get(directory)!; + const ready = await Promise.all(node.dependencies.map(visit)); + const unavailable = node.dependencies.find((_, index) => !ready[index]); + if (unavailable) { + blockers.push({ directory, phase: "dependency", message: `Waiting for ${unavailable}` }); return false; } - const root = path.join(context.workbench, directory); - let phase: AuthoringBlocker["phase"] = "source"; - try { - const lock = await loadQuixosLock(path.join(root, "quixos.lock")); - if (!lock.ok) throw new Error("Lock changed during convergence; retry after joining writers"); - for (const file of lock.lock.sourceFiles ?? ["quixos.lock"]) { - const filename = path.join(root, file), - before = await readFile(filename, "utf8"); - const parsed = parseQuixosLockDocument(before, file); - if (!parsed.ok) throw new Error(`Invalid lock ${file}`); - let changed = false; - for (const dependency of parsed.document.resources) { - const target = selected.get(identity(dependency.kind, dependency.source.repository)); - const source = target ? complete.get(target) : undefined; - if (target && !source) - throw new Error(`Dependencies changed during convergence (${dependency.binding}); join writers and retry`); - if (source && source.commit !== dependency.source.commit) { - dependency.source = source; - changed = true; + return await capture(async () => { + const root = path.join(context.workbench, directory); + let phase: AuthoringBlocker["phase"] = "source"; + try { + const lock = await loadQuixosLock(path.join(root, "quixos.lock")); + if (!lock.ok) throw new Error("Lock changed during convergence; retry after joining writers"); + for (const file of lock.lock.sourceFiles ?? ["quixos.lock"]) { + const filename = path.join(root, file), + before = await readFile(filename, "utf8"); + const parsed = parseQuixosLockDocument(before, file); + if (!parsed.ok) throw new Error(`Invalid lock ${file}`); + let changed = false; + for (const dependency of parsed.document.resources) { + const target = selected.get(identity(dependency.kind, dependency.source.repository)); + const source = target ? complete.get(target) : undefined; + if (target && !source) + throw new Error( + `Dependencies changed during convergence (${dependency.binding}); join writers and retry`, + ); + if (source && source.commit !== dependency.source.commit) { + dependency.source = source; + changed = true; + } + } + if (changed) { + const temporary = `${filename}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, formatQuixosLockDocument(parsed.document), { flag: "wx" }); + if ((await readFile(filename, "utf8")) !== before) + throw new Error(`Concurrent edit to ${file}; retry after joining writers`); + await rename(temporary, filename); + } finally { + await rm(temporary, { force: true }); + } + } } - } - if (changed) { - const temporary = `${filename}.${randomUUID()}.tmp`; - try { - await writeFile(temporary, formatQuixosLockDocument(parsed.document), { flag: "wx" }); - if ((await readFile(filename, "utf8")) !== before) - throw new Error(`Concurrent edit to ${file}; retry after joining writers`); - await rename(temporary, filename); - } finally { - await rm(temporary, { force: true }); + const snapshotStarted = performance.now(); + const commit = await snapshotCommit(root); + if (process.env.QUIXOS_TRACE_CAPTURE === "1") + console.error( + `[${new Date().toISOString()}] Capture: ${directory} snapshot: ${Math.round(performance.now() - snapshotStarted)}ms`, + ); + phase = "publication"; + const ref = retentionTagForCommit(commit); + const remote = await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref]); + if (remote && remote.split(/\s+/)[0] !== commit) + throw new Error(`Conflicting immutable retention ref ${ref}`); + if (!remote) { + await command(root, "git", ["push", node.source.repository, `${commit}:${ref}`]); + if ( + (await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref])).split(/\s+/)[0] !== + commit + ) + throw new Error("Published source retention was not observed"); } + complete.set(directory, { ...node.source, commit }); + } catch (error) { + blockers.push({ directory, phase, message: String(error).slice(0, 4000) }); } - } - const snapshotStarted = performance.now(); - const commit = await snapshotCommit(root); - if (process.env.QUIXOS_TRACE_CAPTURE === "1") - console.error( - `[${new Date().toISOString()}] Capture: ${directory} snapshot: ${Math.round(performance.now() - snapshotStarted)}ms`, - ); - phase = "publication"; - const ref = retentionTagForCommit(commit); - const remote = await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref]); - if (remote && remote.split(/\s+/)[0] !== commit) throw new Error(`Conflicting immutable retention ref ${ref}`); - if (!remote) await command(root, "git", ["push", node.source.repository, `${commit}:${ref}`]); - if ((await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref])).split(/\s+/)[0] !== commit) - throw new Error("Published source retention was not observed"); - complete.set(directory, { ...node.source, commit }); - } catch (error) { - blockers.push({ directory, phase, message: String(error).slice(0, 4000) }); - } - active.delete(directory); - return complete.has(directory); + return complete.has(directory); + }); + })(); + pending.set(directory, result); + return result; }; // Include newly created, not-yet-imported resources, then the root. - if (target === "root") for (const directory of [...nodes.keys()].filter((d) => d !== "root")) await visit(directory); + if (target === "root") await Promise.all([...nodes.keys()].filter((d) => d !== "root").map(visit)); await visit(target); for (let index = blockers.length - 1; index >= 0; index--) if (!visited.has(blockers[index].directory)) blockers.splice(index, 1); - for (const [directory, source] of complete) { + const recheckStarted = performance.now(); + await mapConcurrent([...complete], async ([directory, source]) => { try { if ((await snapshotCommit(path.join(context.workbench, directory))) !== source.commit) throw new Error("Source advanced while converging; join writers and retry"); } catch (error) { blockers.push({ directory, phase: "concurrent-edit", message: String(error) }); } - } + }); + profile("capture-recheck", recheckStarted, { repositories: complete.size }); // Persist successful selections even if another repository is still broken. // Recovery must not depend on all parents succeeding in the same invocation. const graph = JSON.parse(graphBefore); @@ -238,11 +304,14 @@ export async function convergeAuthoring(start: string, target = "root") { await rm(temporary, { force: true }); } } + profile("capture", started, { repositories: nodes.size, snapshots: complete.size * 2, target }); return { workbench: context.workbench, converged: blockers.length === 0, candidate: blockers.length ? null : (complete.get(target) ?? null), - retained: [...complete].map(([directory, source]) => ({ directory, source })), + retained: [...complete] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([directory, source]) => ({ directory, source })), worklist: blockers, verificationEvidence: false, activated: false, diff --git a/src/capability-language/checked-build.ts b/src/capability-language/checked-build.ts index ee04814..7e2e7fd 100644 --- a/src/capability-language/checked-build.ts +++ b/src/capability-language/checked-build.ts @@ -13,13 +13,14 @@ export const checkerIdentity = () => export async function snapshotCommit(root: string): Promise { const run = async (...args: string[]) => (await execFile("jj", args, { cwd: root, env: environment() })).stdout.trim(); - await run("status"); - // jj resolve --list exits 1 on a clean revision. Query structured revision - // metadata instead of depending on diagnostic wording or swallowing errors. - if ((await run("--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "conflict")) !== "false") - throw new Error("Resolve source conflicts before verification"); - const commit = await run("--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "commit_id"); - if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit)) throw new Error("Verification requires an exact jj commit"); + // A single normal log snapshots the working copy and observes both values + // from that exact revision. Separate status/log processes add no evidence. + const [conflict, commit] = (await run("log", "--no-graph", "-r", "@", "-T", 'conflict ++ "\\n" ++ commit_id')).split( + "\n", + ); + if (conflict !== "false") throw new Error("Resolve source conflicts before verification"); + if (!commit || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit)) + throw new Error("Verification requires an exact jj commit"); await execFile("git", ["diff", "--exit-code", "--no-ext-diff", "--no-textconv", commit, "--"], { cwd: root, env: environment(), @@ -110,6 +111,7 @@ export async function buildImmutableCandidate( kind: "workspace" | "interface" | "package" | "build", logFile: string, contractOnly = false, + observe?: (phase: "nixEvaluationMs" | "nixBuildMs", ms: number) => void, ): Promise { if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(source.commit)) throw new Error("An immutable candidate requires an exact commit"); @@ -126,16 +128,52 @@ export async function buildImmutableCandidate( log.once("open", () => resolve()); log.once("error", reject); }); - return await new Promise((resolve, reject) => { - let output = "", - tail = "", - failure: Error | undefined; - const child = spawn( - "nix", - [ - "build", - "--impure", - "--file", + const run = async (executable: string, args: string[]) => + await new Promise((resolve, reject) => { + let output = "", + tail = "", + failure: Error | undefined; + const child = spawn(executable, args, { env: environment(), stdio: ["ignore", "pipe", "pipe"] }); + const failedLog = (error: Error) => { + failure = error; + child.kill(); + }; + log.on("error", failedLog); + child.stdout.on("data", (chunk) => { + output += chunk; + }); + child.stderr.on("data", (chunk) => { + log.write(chunk); + tail = (tail + String(chunk)).slice(-6000); + }); + child.on("error", (error) => { + failure = error; + }); + child.on("close", (code) => { + log.off("error", failedLog); + if (failure) reject(failure); + else if (code !== 0) reject(new Error(`Candidate Nix check failed (${code}). Full log: ${logFile}\n${tail}`)); + else resolve(output.trim()); + }); + }); + const timed = async (phase: "nixEvaluationMs" | "nixBuildMs", action: () => Promise) => { + const started = performance.now(); + try { + return await action(); + } finally { + observe?.(phase, Math.round(performance.now() - started)); + } + }; + let gcDirectory: string | undefined; + try { + // Evaluation includes import-from-derivation builds (locks and contracts). + // Build the resulting derivation directly, without evaluating the DAG twice. + // Keep the derivation alive across the two processes, including concurrent + // GC. This temporary root is removed on success and on verification failure. + gcDirectory = await fs.mkdtemp(path.join(path.dirname(logFile), ".gc-root-")); + const gcRoot = path.join(gcDirectory, "candidate"); + const derivation = await timed("nixEvaluationMs", async () => { + await run("nix-instantiate", [ builder, "--argstr", "repository", @@ -152,33 +190,22 @@ export async function buildImmutableCandidate( "--arg", "contractOnly", contractOnly ? "true" : "false", - "--no-link", - "--print-out-paths", - "-L", - ], - { env: environment(), stdio: ["ignore", "pipe", "pipe"] }, + "--add-root", + gcRoot, + "--indirect", + ]); + return await fs.readlink(gcRoot); + }); + if (!/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+\.drv$/.test(derivation)) + throw new Error("Nix returned an invalid candidate derivation"); + const artifact = await timed("nixBuildMs", () => + run("nix", ["build", derivation + "^*", "--no-link", "--print-out-paths", "-L"]), ); - log.on("error", (error) => { - failure = error; - child.kill(); - }); - child.stdout.on("data", (chunk) => { - output += chunk; - }); - child.stderr.on("data", (chunk) => { - log.write(chunk); - tail = (tail + String(chunk)).slice(-6000); - }); - child.on("error", (error) => { - failure = error; - }); - child.on("close", (code) => - log.end(() => { - if (failure) reject(failure); - else if (code !== 0 || !/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+$/.test(output.trim())) - reject(new Error(`Candidate Nix check failed (${code}). Full log: ${logFile}\n${tail}`)); - else resolve(output.trim()); - }), - ); - }); + if (!/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+$/.test(artifact)) + throw new Error("Nix returned an invalid candidate artifact"); + return artifact; + } finally { + await new Promise((resolve) => log.end(resolve)); + if (gcDirectory) await fs.rm(gcDirectory, { recursive: true, force: true }); + } } diff --git a/src/capability-language/checked-resources.ts b/src/capability-language/checked-resources.ts new file mode 100644 index 0000000..1d258fa --- /dev/null +++ b/src/capability-language/checked-resources.ts @@ -0,0 +1,40 @@ +import { readFile } from "node:fs/promises"; +import type { CheckedResourceResolver } from "./assembly.js"; +import type { CapabilityResource } from "./parser.js"; + +/** Internal compiler inputs produced by checked-candidate.nix, not authoring + * receipts. The Nix composition owns their derivations and exact toolchain. + * Ordinary compiler callers omit this map and run all checks directly. */ +export async function loadCheckedResources(filename?: string): Promise { + if (!filename) return undefined; + const entries = JSON.parse(await readFile(filename, "utf8")) as Array<{ + kind: string; + repository: string; + commit: string; + artifact: string; + }>; + const resources = new Map(); + for (const entry of entries) { + const key = `${entry.kind}\0${entry.repository}\0${entry.commit}`; + if (resources.has(key)) throw new Error("Repeated checked resource input"); + if (!/^\/nix\/store\/[^/]+$/.test(entry.artifact)) + throw new Error("Checked resources must be immutable Nix outputs"); + resources.set(key, entry.artifact); + } + return async (kind, source, lock, dependencies) => { + const artifact = resources.get(`${kind}\0${source.repository}\0${source.commit}`); + if (!artifact) return undefined; + const graph = JSON.parse(await readFile(`${artifact}/graph.json`, "utf8")); + const expected = [...dependencies].map(([binding, node]) => ({ binding, resourceKey: node.key })); + if ( + graph.quixos.commit !== lock.quixos.commit || + graph.quixos.repository !== lock.quixos.repository || + graph.root.kind !== kind || + graph.root.source.repository !== source.repository || + graph.root.source.commit !== source.commit || + JSON.stringify(graph.root.dependencies) !== JSON.stringify(expected) + ) + throw new Error("Checked resource input does not match its locked dependency graph"); + return JSON.parse(await readFile(`${artifact}/candidate.json`, "utf8")) as CapabilityResource; + }; +} diff --git a/src/capability-language/execution-scaffold.ts b/src/capability-language/execution-scaffold.ts index e690ac8..226154e 100644 --- a/src/capability-language/execution-scaffold.ts +++ b/src/capability-language/execution-scaffold.ts @@ -80,6 +80,7 @@ export async function executionScaffold( devShells.default = (import (helpers + "/quixos-package-helpers.nix")).mkCaminoYarnShell { inherit pkgs; }; quixosPackages.checkedArtifacts = context: let + pkgs = camino.packageSet { inherit nixpkgs system context; }; contract = builtins.fromJSON (builtins.readFile context.buildContract); package = contract.definition.package; portable = builtins.filter (entry: entry.execution.kind != "remote") package.exports; diff --git a/src/capability-language/profile.ts b/src/capability-language/profile.ts new file mode 100644 index 0000000..5ff6c79 --- /dev/null +++ b/src/capability-language/profile.ts @@ -0,0 +1,5 @@ +/** Observations belong in logs, never in content-addressed compiler outputs. */ +export const profile = (phase: string, started: number, details: Record = {}) => { + if (process.env.QUIXOS_TRACE_PROFILE === "1") + console.error(`[qx-profile] ${JSON.stringify({ phase, ms: Math.round(performance.now() - started), ...details })}`); +}; diff --git a/src/capability-language/resource-cli.ts b/src/capability-language/resource-cli.ts index 836419a..4f40494 100644 --- a/src/capability-language/resource-cli.ts +++ b/src/capability-language/resource-cli.ts @@ -6,11 +6,12 @@ import { bindingSchema } from "../bindings/index.js"; import { compileCapabilityResourceRepository, type ResolvedCapabilityResource } from "./assembly.js"; import { createGitCapabilityResolver } from "./git-resolver.js"; import { loadQueryGenerator } from "./query-generator.js"; +import { loadCheckedResources } from "./checked-resources.js"; const usage = `usage: quixos-resource-compile --root DIRECTORY --kind interface|package|build --repository URL --commit GIT_REV --checkout-root DIRECTORY [--snapshot-map PATH] [--snapshot-only true] [--graph-out PATH] [--schema-out PATH] - [--query-generator BUILD_MODULE_PATH] + [--query-generator BUILD_MODULE_PATH] [--checked-resources NIX_MAP] Resolves a standalone resource's recursive dependency-lock graph, validates its interface.qx or package.qx manifest, and emits the checked resource as JSON.`; @@ -33,6 +34,7 @@ const parseArgs = (args: string[]) => { "--graph-out", "--schema-out", "--query-generator", + "--checked-resources", ].includes(key) || values.has(key) ) @@ -69,6 +71,7 @@ const parseArgs = (args: string[]) => { graphOut: values.get("--graph-out"), schemaOut: values.get("--schema-out"), queryGenerator: values.get("--query-generator"), + checkedResources: values.get("--checked-resources"), } as const; }; @@ -111,6 +114,7 @@ const main = async () => { }, resolveResource, generateQueries: await loadQueryGenerator(options.queryGenerator), + checkedResource: await loadCheckedResources(options.checkedResources), }); if (options.graphOut) { await writeFile( diff --git a/src/capability-language/workspace-cli.ts b/src/capability-language/workspace-cli.ts index 18b2ec5..1159d8c 100644 --- a/src/capability-language/workspace-cli.ts +++ b/src/capability-language/workspace-cli.ts @@ -5,6 +5,8 @@ import process from "node:process"; import { compileWorkspaceRepository } from "./assembly.js"; import { createGitCapabilityResolver } from "./git-resolver.js"; import { loadQueryGenerator } from "./query-generator.js"; +import { loadCheckedResources } from "./checked-resources.js"; +import { profile } from "./profile.js"; import { compileExecutionPlan, packageBuildContexts } from "../execution-world/index.js"; import { bindingSchema, specializeBindingSchema } from "../bindings/index.js"; import { @@ -18,7 +20,7 @@ const usage = `usage: quixos-workspace-compile --root DIRECTORY --checkout-root [--snapshot-map PATH] [--graph-out PATH] [--workspace-id ID] [--workspace-revision-id ID] [--source-root-commit GIT_REV] [--baseline PLAN_JSON] [--evolution-out PATH] [--reviews REVIEW_JSON] [--schemas-out PATH] [--execution-plan-out PATH] [--build-contexts-out PATH] - [--query-generator BUILD_MODULE_PATH] + [--query-generator BUILD_MODULE_PATH] [--checked-resources NIX_MAP] Resolves a workspace's recursive resource-lock graph, clones every exact resource revision, validates standalone interface/package manifests, and emits @@ -50,6 +52,7 @@ const parseArgs = (args: string[]) => { evolutionOut: values.get("--evolution-out"), reviews: values.get("--reviews"), queryGenerator: values.get("--query-generator"), + checkedResources: values.get("--checked-resources"), }; }; @@ -63,6 +66,7 @@ const main = async () => { checkoutRoot: options.checkoutRoot, snapshotMap: options.snapshotMap, }); + const assemblyStarted = performance.now(); const assembled = await compileWorkspaceRepository({ rootDirectory: options.rootDirectory, workspaceId: options.workspaceId, @@ -70,7 +74,9 @@ const main = async () => { sourceRootCommit: options.sourceRootCommit, resolveResource, generateQueries: await loadQueryGenerator(options.queryGenerator), + checkedResource: await loadCheckedResources(options.checkedResources), }); + profile("workspace-assembly", assemblyStarted, { resources: assembled.resources.length }); if (options.graphOut) { await writeFile( @@ -110,12 +116,15 @@ const main = async () => { } const candidate = { ...assembled.workspace, executionContracts: runtimeContracts(assembled.workspace) }; if (options.executionPlanOut || options.buildContextsOut) { + const executionStarted = performance.now(); const plan = compileExecutionPlan(assembled.workspace); if (options.executionPlanOut) await writeFile(options.executionPlanOut, `${JSON.stringify(plan, null, 2)}\n`); if (options.buildContextsOut) await writeFile(options.buildContextsOut, JSON.stringify(packageBuildContexts(plan.definition.bindingSchema))); + profile("execution-plan-and-contexts", executionStarted); } if (options.schemasOut) { + const schemasStarted = performance.now(); const schemas: Record = {}; for (const node of assembled.capabilityResources.filter((entry) => entry.kind === "package")) { const closure = new Map(); @@ -133,6 +142,7 @@ const main = async () => { ); } await writeFile(options.schemasOut, JSON.stringify(schemas)); + profile("package-schemas", schemasStarted); } if (options.evolutionOut) { const baseline = options.baseline diff --git a/test/capability-assembly.test.ts b/test/capability-assembly.test.ts index c147e19..33a16b1 100644 --- a/test/capability-assembly.test.ts +++ b/test/capability-assembly.test.ts @@ -107,6 +107,35 @@ package Runtime id "package:runtime" revision "package:runtime@1" { assert.equal(resource.resource.kind, "package"); assert.equal(resource.resources.length, 2); assert.equal(resource.directResources.size, 1); + + const checked = new Map(result.resources.map((node) => [node.key, node.resource])); + let generated = 0; + const reused = await compileWorkspaceRepository({ + rootDirectory: root, + resolveResource: async (source, kind) => ({ + directory: directories.get(`${kind}\0${source.repository}\0${source.commit}`)!, + }), + checkedResource: async (kind, source) => { + const value = checked.get(`${kind}\0${source.repository}\0${source.commit}`); + return value?.kind === "build" ? undefined : value; + }, + generateQueries: async () => { + generated++; + return { queries: [], documents: new Map() }; + }, + }); + assert.deepEqual(reused, result, "Nix-composed resource checks preserve full assembly results"); + assert.equal(generated, 0, "unchanged checked packages do not run their query frontend again"); + await assert.rejects( + compileWorkspaceRepository({ + rootDirectory: root, + resolveResource: async (source, kind) => ({ + directory: directories.get(`${kind}\0${source.repository}\0${source.commit}`)!, + }), + checkedResource: async () => (resource.resource.kind === "build" ? undefined : resource.resource), + }), + /Checked resource identity mismatch/, + ); }); test("resource repositories cannot own workspace Quixos selection policy", async (context) => { @@ -249,19 +278,48 @@ test("resource-owned fragment imports compile through the locked package closure }`, ); await writeFile(path.join(dirs.screen!, "screen.graphql"), "query Screen {root {...Row}}"); - const result = await compileCapabilityResourceRepository({ + const options = { rootDirectory: dirs.screen!, - kind: "package", - source: { resolver: "git", repository: "https://example.test/screen.git", commit: packageCommit }, - resolveResource: async (source) => { + kind: "package" as const, + source: { resolver: "git" as const, repository: "https://example.test/screen.git", commit: packageCommit }, + resolveResource: async (source: { repository: string }) => { const name = new URL(source.repository).pathname.slice(1, -4); assert.ok(dirs[name], name); return { directory: dirs[name]! }; }, - }); + }; + const result = await compileCapabilityResourceRepository(options); assert.equal(result.resource.kind, "package"); if (result.resource.kind !== "package") throw Error("expected package"); const view = result.resource.revision.checkedQueries!.find((q) => q.declaration.id === "view")!; assert.equal(view.selection[0]!.selection[0]!.fragmentSpread?.package, "rows@1"); assert.equal(result.resources.length, 3); + const checked = new Map(result.resources.map((node) => [node.key, node.resource])); + let generated = 0; + const reused = await compileCapabilityResourceRepository({ + ...options, + checkedResource: async (kind, source) => { + if (source.repository === options.source.repository) return undefined; + const resource = checked.get(`${kind}\0${source.repository}\0${source.commit}`); + return resource?.kind === "build" ? undefined : resource; + }, + generateQueries: async () => { + generated++; + return { queries: [], documents: new Map() }; + }, + }); + assert.deepEqual(reused, result, "cached imported fragments and full checking produce identical contracts"); + assert.equal(generated, 1, "only the uncached package checks its queries"); + await writeFile(path.join(dirs.screen!, "screen.graphql"), "query Screen {root {unknownField}}"); + await assert.rejects( + compileCapabilityResourceRepository({ + ...options, + source: { ...options.source, commit: "5".repeat(40) }, + checkedResource: async (kind, source) => { + const resource = checked.get(`${kind}\0${source.repository}\0${source.commit}`); + return resource?.kind === "build" ? undefined : resource; + }, + }), + /unknownField/, + ); });