Instrument and improve incremental workspace checks

Profile source capture, Nix evaluation/realization, resource/query checking,
and implementation builds without putting timings into immutable outputs.
Keep modular repositories; bound independent capture work, snapshot jj once
per observation, and retain final concurrent-edit and Git completeness checks.

Compose exact resource/closure/frontend/toolchain checks as memoized Nix
derivations. Preserve whole-workspace validation and the full compiler path
when no checked dependency is supplied. Share canonical Nixpkgs by content,
index workspace build contexts once, and narrow lock/driver/compiler inputs
with conservative complete-source fallback. Batch React witness checks only
when distinct source closures cannot share ambient declarations; otherwise
run the original independent TypeScript checks.

Seed local authoring toolchains with exact canonical fetch identities when
they match the selected inputs; retain store-path inputs for overrides.
Add generic correctness checks rather than permanent Reader timing/content
fixtures. Verify actual Reader edits and rejection/recovery in a private
workbench, preserving the original user's workbench and runtime.

Query checking currently caches at resource/closure boundaries, not individual
query semantic dependency boundaries. Template helper pins and reproducible
fixture snapshots follow in a separate publication change.
This commit is contained in:
Timothy J. Aveni
2026-10-01 04:03:19 -07:00
parent e8d19d2bb7
commit 7c1830bcd5
15 changed files with 527 additions and 141 deletions
+3 -1
View File
@@ -81,11 +81,13 @@
install -Dm644 nix/checked-package.nix "$out/share/checked-package.nix"
install -Dm644 nix/checked-artifacts.nix "$out/share/checked-artifacts.nix"
install -Dm644 nix/package-context.nix "$out/share/package-context.nix"
install -Dm644 nix/lock-source.nix "$out/share/lock-source.nix"
install -Dm644 nix/package-context.mjs "$out/share/package-context.mjs"
install -Dm644 nix/check-package.mjs "$out/share/check-package.mjs"
install -Dm644 nix/check-execution.mjs "$out/share/check-execution.mjs"
substitute nix/checked-candidate.nix "$out/share/checked-candidate.nix" \
--replace-fail '@nixpkgs@' '${pkgs.path}'
--replace-fail '@nixpkgs@' '${pkgs.path}' \
--replace-fail '@nixpkgsNarHash@' '${nixpkgs.narHash}'
cp --reflink=auto --recursive grammar "$out/grammar"
cp --reflink=auto --recursive proto "$out/proto"
cp --reflink=auto --recursive dist "$out/dist"
+62 -12
View File
@@ -9,7 +9,11 @@
system ? builtins.currentSystem,
}:
let
pkgs = import (/. + "@nixpkgs@") { inherit system; };
pkgs = import (/. + "@nixpkgs@") {
inherit system;
config = { };
overlays = [ ];
};
protocol = builtins.storePath generator;
sourceKey = source: "${source.kind}:${source.repository}@${source.commit}";
fetch =
@@ -26,8 +30,9 @@ let
source:
let
directory = fetch source;
lockSource = import ./lock-source.nix { source = directory; };
lockFile = pkgs.runCommand "qx-source-lock.json" { } ''
${protocol}/bin/quixos-lock-check ${directory}/quixos.lock > "$out"
${protocol}/bin/quixos-lock-check ${lockSource}/quixos.lock > "$out"
'';
lock = builtins.fromJSON (builtins.readFile lockFile);
in
@@ -124,15 +129,40 @@ let
}) (builtins.filter (entry: entry.kind != "workspace") (resourceClosure true node));
}
);
compile =
# Each resource is checked once per exact source/closure/frontend/toolchain.
# Composition imports these compiler outputs; authoring receipts are never
# build inputs. Missing entries in ordinary CLI use take the full check path.
compiledNodes = builtins.mapAttrs (_: node: compileNode node) nodeTable;
compile = node: compiledNodes.${sourceKey node};
checkedResources =
node:
pkgs.runCommand "qx-${node.kind}-contract" { } ''
pkgs.writeText "qx-checked-resources.json" (
builtins.toJSON (
map
(dependency: {
inherit (dependency) kind repository commit;
artifact = compile dependency;
})
(
builtins.filter (
dependency:
dependency.kind != "workspace"
&& dependency.kind != "build"
&& sourceKey dependency != sourceKey node
) (resourceClosure true node)
)
)
);
compileNode =
node:
pkgs.runCommand "qx-${node.kind}-contract" { QUIXOS_TRACE_PROFILE = "1"; } ''
mkdir -p "$out"
${
if node.kind == "workspace" then
''
${protocol}/bin/quixos-workspace-compile --root ${node.directory} \
${queryGeneratorFlags node} \
${queryGeneratorFlags root} \
--checked-resources ${checkedResources node} \
--source-root-commit ${pkgs.lib.escapeShellArg node.commit} \
--checkout-root "$TMPDIR/checkouts" --snapshot-map ${snapshots node} \
--graph-out "$out/graph.json" --schemas-out "$out/package-schemas.json" \
@@ -144,7 +174,8 @@ let
else
''
${protocol}/bin/quixos-resource-compile --root ${node.directory} \
${queryGeneratorFlags node} \
${queryGeneratorFlags root} \
--checked-resources ${checkedResources node} \
--kind ${node.kind} --repository ${pkgs.lib.escapeShellArg node.repository} \
--commit ${pkgs.lib.escapeShellArg node.commit} \
--checkout-root "$TMPDIR/checkouts" --snapshot-map ${snapshots node} --snapshot-only true \
@@ -153,6 +184,18 @@ let
}
'';
contract = compile root;
# Parse shared workspace outputs once. Repeating fromJSON in every package
# makes composition quadratic in package count and in the serialized world.
workspacePackages = builtins.listToAttrs (
map
(pkg: {
name = sourceKey (pkg.source // { kind = "package"; });
value = pkg;
})
(builtins.fromJSON (builtins.readFile "${contract}/execution-plan.json"))
.definition.bindingSchema.packages
);
workspaceContexts = builtins.fromJSON (builtins.readFile "${contract}/build-contexts.json");
buildDependencies =
node:
builtins.listToAttrs (
@@ -218,17 +261,19 @@ let
if execution && kind == "workspace" then
{
revision =
pkgs.lib.findFirst
(pkg: pkg.source.repository == node.repository && pkg.source.commit == node.commit)
(throw "Source package missing from workspace compilation: ${node.repository}")
(builtins.fromJSON (builtins.readFile "${contract}/execution-plan.json"))
.definition.bindingSchema.packages;
workspacePackages.${sourceKey node}
or (throw "Source package missing from workspace compilation: ${node.repository}");
}
else
builtins.fromJSON (builtins.readFile "${compiled}/candidate.json");
buildContext = import ./package-context.nix {
inherit pkgs protocol;
contexts = if kind == "workspace" then "${contract}/build-contexts.json" else null;
contextData =
if execution && kind == "workspace" then
workspaceContexts.${candidate.revision.revisionId}
else
null;
plan = "${compiled}/bindings.json";
packageRevisionId = candidate.revision.revisionId;
};
@@ -254,6 +299,11 @@ let
typescript = runtimePackages.camino-replica-core.dependencies;
packageRevisionId = candidate.revision.revisionId;
dependencies = buildDependencies node;
buildEnvironment = {
inherit pkgs system;
nixpkgsSource = toString pkgs.path;
narHash = "@nixpkgsNarHash@";
};
}
// pkgs.lib.optionalAttrs (componentEntries != [ ]) {
react = {
@@ -310,7 +360,7 @@ in
pkgs.runCommand (if contractOnly then "qx-contract" else "qx-checked-candidate") { } ''
mkdir -p "$out"
cp ${
pkgs.writeText "qx-candidate-source.json" (builtins.toJSON { inherit repository commit kind; })
builtins.toFile "qx-candidate-source.json" (builtins.toJSON { inherit repository commit kind; })
} "$out/source.json"
cp ${contract}/candidate.json "$out/candidate.json"
cp ${contract}/graph.json "$out/graph.json"
+16
View File
@@ -0,0 +1,16 @@
# Lock parsing only reads this one file when no import token can occur. A
# conservative substring probe also sends comments/URLs containing "import"
# through the complete-source path. Fragment names may have any extension.
{ source }:
let
regular = ((builtins.readDir source)."quixos.lock" or null) == "regular";
text = if regular then builtins.readFile (source + "/quixos.lock") else "import";
in
if builtins.replaceStrings [ "import" ] [ "" ] text != text then
source
else
builtins.path {
path = source;
name = "qx-lock-source";
filter = filename: _: filename == toString source || filename == "${source}/quixos.lock";
}
+42
View File
@@ -0,0 +1,42 @@
{ pkgs, protocol }:
let
base = ''quixos-lock version 1 { quixos source { repository "https://example.test/quixos.git"; commit "1111111111111111111111111111111111111111"; } }'';
fixture =
extra: lock:
pkgs.runCommand "qx-lock-source-fixture" { } ''
mkdir -p "$out/nested"
printf '%s' ${pkgs.lib.escapeShellArg lock} > "$out/quixos.lock"
printf '%s' ${pkgs.lib.escapeShellArg extra} > "$out/implementation.ts"
printf '%s' 'quixos-lock fragment version 1 {}' > "$out/nested/dependencies.txt"
'';
first = fixture "first implementation" base;
edited = fixture "edited implementation" base;
changed = fixture "first implementation" (
builtins.replaceStrings
[ "1111111111111111111111111111111111111111" ]
[ "2222222222222222222222222222222222222222" ]
base
);
fragments = fixture "first implementation" (
builtins.replaceStrings [ "quixos source" ] [ ''import "nested/dependencies.txt"; quixos source'' ]
base
);
conservative = fixture "first implementation" (base + "\n// import in a comment\n");
slice = source: import ./lock-source.nix { inherit source; };
parse =
source:
pkgs.runCommand "qx-source-lock.json" { } ''
${protocol}/bin/quixos-lock-check ${slice source}/quixos.lock > "$out"
'';
in
assert slice first == slice edited;
assert (parse first).drvPath == (parse edited).drvPath;
assert slice first != slice changed;
pkgs.runCommand "qx-lock-source-check" { } ''
${protocol}/bin/quixos-lock-check ${slice first}/quixos.lock > first.json
${protocol}/bin/quixos-lock-check ${slice edited}/quixos.lock > edited.json
cmp first.json edited.json
${protocol}/bin/quixos-lock-check ${slice conservative}/quixos.lock > comment.json
cmp first.json comment.json
${protocol}/bin/quixos-lock-check ${slice fragments}/quixos.lock > "$out"
''
+4 -1
View File
@@ -5,6 +5,7 @@
protocol,
plan ? null,
contexts ? null,
contextData ? null,
packageRevisionId,
}:
let
@@ -14,7 +15,9 @@ let
${pkgs.lib.escapeShellArg packageRevisionId} > "$out"
'';
context =
if contexts == null then
if contextData != null then
contextData
else if contexts == null then
builtins.fromJSON (builtins.readFile projected)
else
(builtins.fromJSON (builtins.readFile contexts)).${packageRevisionId};
+57 -1
View File
@@ -1,5 +1,6 @@
import { readFile, realpath } from "node:fs/promises";
import path from "node:path";
import { profile } from "./profile.js";
import { loadQxSources, resolveQxSources } from "./source-loader.js";
import { compileQuery, compileRepositoryQuery } from "../query/compile.js";
import type { QueryDeclaration } from "../query/types.js";
@@ -43,6 +44,15 @@ export type CapabilityRepositoryResolver = (
kind: LockedResource["kind"],
) => Promise<CapabilityRepositorySnapshot>;
/** Already checked outputs of this compiler, composed by the Nix build DAG.
* Source capture/check reports never grant this authority. */
export type CheckedResourceResolver = (
kind: LockedResource["kind"],
source: GitSource,
lock: QuixosRepositoryLock,
dependencies: ReadonlyMap<string, ResolvedCapabilityResource>,
) => Promise<CapabilityResource | undefined>;
/** An explicitly selected build frontend supplies ordinary query sources. The
* assembler never chooses a frontend by inspecting an interface/contract ID.
* Generated sources go through the same query and imported-fragment checks. */
@@ -221,7 +231,12 @@ const createResourceGraphResolver = (
quixosCommit: string,
resolveResource: CapabilityRepositoryResolver,
generateQueries?: PackageQueryGenerator,
checkedResource?: CheckedResourceResolver,
) => {
const graphStarted = performance.now();
let checkedResources = 0,
reusedResources = 0,
checkedQueries = 0;
const resolved = new Map<string, Promise<ResolvedCapabilityResource>>();
const active: string[] = [];
@@ -275,6 +290,30 @@ const createResourceGraphResolver = (
};
}
const dependencyClosure = resourceClosure(directPairs.map(([, node]) => node));
const dependencies = new Map(
directPairs.map(([dependency, node]) => [bindingKey(dependency.kind, dependency.binding), node]),
);
const checked = await checkedResource?.(locked.kind, locked.source, lockResult.lock, dependencies);
if (checked) {
if (
checked.kind !== locked.kind ||
checked.revision.source.repository !== locked.source.repository ||
checked.revision.source.commit !== locked.source.commit
)
throw new Error("Checked resource identity mismatch");
assertImportsMatchLock("Checked resource", checked.imports, lockResult.lock.resources);
reusedResources++;
return {
key,
kind: locked.kind,
source: locked.source,
directory: snapshot.directory,
lock: lockResult.lock,
resource: structuredClone(checked),
dependencies,
};
}
const checkStarted = performance.now();
const environment = environmentFor(directPairs, dependencyClosure);
const manifestName = locked.kind === "interface" ? "interface.qx" : "package.qx";
const manifestPath = path.join(snapshot.directory, manifestName);
@@ -399,6 +438,15 @@ const createResourceGraphResolver = (
compiled.resource.revision.migrationCatalog = catalog;
}
}
checkedResources++;
const queries =
compiled.resource.kind === "package" ? (compiled.resource.revision.checkedQueries?.length ?? 0) : 0;
checkedQueries += queries;
profile("resource-check", checkStarted, {
kind: locked.kind,
resource: compiled.resource.revision.displayName,
queries,
});
return {
key,
kind: locked.kind,
@@ -420,7 +468,11 @@ const createResourceGraphResolver = (
return {
visit,
nodes: async () => await Promise.all(resolved.values()),
nodes: async () => {
const nodes = await Promise.all(resolved.values());
profile("resource-graph", graphStarted, { checkedResources, reusedResources, checkedQueries });
return nodes;
},
};
};
@@ -430,6 +482,7 @@ export const compileCapabilityResourceRepository = async (options: {
source: GitSource;
resolveResource: CapabilityRepositoryResolver;
generateQueries?: PackageQueryGenerator;
checkedResource?: CheckedResourceResolver;
}): Promise<CompiledCapabilityResourceRepository> => {
const lockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock"));
if (!lockResult.ok) {
@@ -439,6 +492,7 @@ export const compileCapabilityResourceRepository = async (options: {
lockResult.lock.quixos.commit,
options.resolveResource,
options.generateQueries,
options.checkedResource,
);
const root = await resolver.visit(
{
@@ -466,6 +520,7 @@ export const compileWorkspaceRepository = async (options: {
/** An editor's proposed source snapshot; locks and dependency revisions remain exact. */
readSource?: (name: string) => Promise<string>;
generateQueries?: PackageQueryGenerator;
checkedResource?: CheckedResourceResolver;
}): Promise<CompiledWorkspaceRepository> => {
const rootLockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock"));
if (!rootLockResult.ok) {
@@ -476,6 +531,7 @@ export const compileWorkspaceRepository = async (options: {
rootLock.quixos.commit,
options.resolveResource,
options.generateQueries,
options.checkedResource,
);
const directPairs: Array<[LockedResource, ResolvedCapabilityResource]> = [];
+4 -1
View File
@@ -64,7 +64,7 @@ export async function checkAuthoring(
// Repository-scoped agents can check separate immutable candidates in parallel.
const capture = promisify(callback)("quixos-qx", ["converge", context.workbench, directory], {
maxBuffer: 4 * 1024 * 1024,
env: { ...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_TRACE_CAPTURE: "1" },
env: { ...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_TRACE_CAPTURE: "1", QUIXOS_TRACE_PROFILE: "1" },
});
console.error(`Capture details: ${path.join(output, "capture.log")}`);
capture.child.stderr?.on("data", (chunk) => appendFileSync(path.join(output, "capture.log"), chunk));
@@ -94,6 +94,9 @@ export async function checkAuthoring(
resource.kind,
path.join(output, "nix.log"),
options.contractOnly,
(phase, ms) => {
timings[phase] = ms;
},
);
} finally {
timings.immutableCheckMs = Math.round(performance.now() - buildStarted);
+142 -73
View File
@@ -5,6 +5,7 @@ import { promisify } from "node:util";
import { randomUUID } from "node:crypto";
import { authoringContext } from "./authoring-context.js";
import { snapshotCommit } from "./checked-build.js";
import { profile } from "./profile.js";
import {
loadQuixosLock,
parseQuixosLockDocument,
@@ -14,6 +15,42 @@ import {
} from "../resource-lock/index.js";
const execFile = promisify(callback);
// Bound independent repository operations. One visit owns each repository;
// all visits finish before the final concurrent-edit validation pass.
const concurrency = 4;
async function mapConcurrent<T, R>(values: readonly T[], action: (value: T) => Promise<R>): Promise<R[]> {
const results = new Array<R>(values.length);
let next = 0;
await Promise.all(
Array.from({ length: Math.min(concurrency, values.length) }, async () => {
while (next < values.length) {
const index = next++;
results[index] = await action(values[index]);
}
}),
);
return results;
}
const limiter = () => {
let running = 0;
const waiting: Array<() => void> = [];
return async <T>(action: () => Promise<T>): Promise<T> => {
await new Promise<void>((resolve) => {
if (running < concurrency) {
running++;
resolve();
} else waiting.push(resolve);
});
try {
return await action();
} finally {
const next = waiting.shift();
if (next) next();
else running--;
}
};
};
const command = async (cwd: string, executable: string, args: string[]) => {
const start = performance.now();
// Do not log arguments: transports may contain credentials. Source identities
@@ -44,13 +81,14 @@ export type AuthoringBlocker = {
* worklist grants typechecking, semantic review or activation approval.
* Caller serializes coordinators; package authors may still be editing. */
export async function convergeAuthoring(start: string, target = "root") {
const started = performance.now();
const context = await authoringContext(start);
const graphFile = path.join(context.workbench, ".quixos/resource-graph.json");
const graphBefore = await readFile(graphFile, "utf8");
const blockers: AuthoringBlocker[] = [];
const nodes = new Map<string, { directory: string; kind: string; source: GitSource; dependencies: string[] }>();
const selected = new Map<string, string>();
for (const entry of context.resources) {
const inventory = await mapConcurrent(context.resources, async (entry) => {
const root = path.join(context.workbench, entry.directory);
let repository = entry.source?.repository;
try {
@@ -64,6 +102,9 @@ export async function convergeAuthoring(start: string, target = "root") {
blockers.push({ directory: entry.directory, phase: "source", message: String(error).slice(0, 2000) });
if (!repository) throw error; // The root has no separate registered source.
}
return { ...entry, repository };
});
for (const { repository, ...entry } of inventory) {
const key = identity(entry.kind, repository);
if (selected.has(key)) throw new Error(`More than one editable checkout for ${repository}`);
selected.set(key, entry.directory);
@@ -73,7 +114,7 @@ export async function convergeAuthoring(start: string, target = "root") {
dependencies: [],
});
}
for (const node of nodes.values()) {
await mapConcurrent([...nodes.values()], async (node) => {
try {
const lock = await loadQuixosLock(path.join(context.workbench, node.directory, "quixos.lock"));
if (!lock.ok) throw new Error(lock.diagnostics.map((d) => `${d.fileName}: ${d.message}`).join("\n"));
@@ -88,97 +129,122 @@ export async function convergeAuthoring(start: string, target = "root") {
} catch (error) {
blockers.push({ directory: node.directory, phase: "resolution", message: String(error) });
}
}
const complete = new Map<string, GitSource>(),
active = new Set<string>();
const visited = new Set<string>();
if (!nodes.has(target)) throw new Error(`Not a registered repository: ${target}`);
const visit = async (directory: string): Promise<boolean> => {
visited.add(directory);
if (complete.has(directory)) return true;
if (blockers.some((entry) => entry.directory === directory)) return false;
if (active.has(directory)) {
});
// Detect cycles before starting concurrent futures: two roots awaiting each
// other must fail clearly rather than deadlocking on their cached promises.
const scanned = new Set<string>();
const detectCycles = (directory: string, parents: string[] = []) => {
if (parents.includes(directory)) {
blockers.push({
directory,
phase: "dependency",
message: `Source dependency cycle: ${[...active, directory].join(" -> ")}`,
message: `Source dependency cycle: ${[...parents, directory].join(" -> ")}`,
});
return false;
return;
}
active.add(directory);
const node = nodes.get(directory)!;
for (const dependency of node.dependencies)
if (!(await visit(dependency))) {
blockers.push({ directory, phase: "dependency", message: `Waiting for ${dependency}` });
active.delete(directory);
if (scanned.has(directory)) return;
for (const dependency of nodes.get(directory)!.dependencies) detectCycles(dependency, [...parents, directory]);
scanned.add(directory);
};
for (const directory of nodes.keys()) detectCycles(directory);
const complete = new Map<string, GitSource>();
const pending = new Map<string, Promise<boolean>>();
const capture = limiter();
const visited = new Set<string>();
if (!nodes.has(target)) throw new Error(`Not a registered repository: ${target}`);
const visit = (directory: string): Promise<boolean> => {
visited.add(directory);
const existing = pending.get(directory);
if (existing) return existing;
const result = (async () => {
if (blockers.some((entry) => entry.directory === directory)) return false;
const node = nodes.get(directory)!;
const ready = await Promise.all(node.dependencies.map(visit));
const unavailable = node.dependencies.find((_, index) => !ready[index]);
if (unavailable) {
blockers.push({ directory, phase: "dependency", message: `Waiting for ${unavailable}` });
return false;
}
const root = path.join(context.workbench, directory);
let phase: AuthoringBlocker["phase"] = "source";
try {
const lock = await loadQuixosLock(path.join(root, "quixos.lock"));
if (!lock.ok) throw new Error("Lock changed during convergence; retry after joining writers");
for (const file of lock.lock.sourceFiles ?? ["quixos.lock"]) {
const filename = path.join(root, file),
before = await readFile(filename, "utf8");
const parsed = parseQuixosLockDocument(before, file);
if (!parsed.ok) throw new Error(`Invalid lock ${file}`);
let changed = false;
for (const dependency of parsed.document.resources) {
const target = selected.get(identity(dependency.kind, dependency.source.repository));
const source = target ? complete.get(target) : undefined;
if (target && !source)
throw new Error(`Dependencies changed during convergence (${dependency.binding}); join writers and retry`);
if (source && source.commit !== dependency.source.commit) {
dependency.source = source;
changed = true;
return await capture(async () => {
const root = path.join(context.workbench, directory);
let phase: AuthoringBlocker["phase"] = "source";
try {
const lock = await loadQuixosLock(path.join(root, "quixos.lock"));
if (!lock.ok) throw new Error("Lock changed during convergence; retry after joining writers");
for (const file of lock.lock.sourceFiles ?? ["quixos.lock"]) {
const filename = path.join(root, file),
before = await readFile(filename, "utf8");
const parsed = parseQuixosLockDocument(before, file);
if (!parsed.ok) throw new Error(`Invalid lock ${file}`);
let changed = false;
for (const dependency of parsed.document.resources) {
const target = selected.get(identity(dependency.kind, dependency.source.repository));
const source = target ? complete.get(target) : undefined;
if (target && !source)
throw new Error(
`Dependencies changed during convergence (${dependency.binding}); join writers and retry`,
);
if (source && source.commit !== dependency.source.commit) {
dependency.source = source;
changed = true;
}
}
if (changed) {
const temporary = `${filename}.${randomUUID()}.tmp`;
try {
await writeFile(temporary, formatQuixosLockDocument(parsed.document), { flag: "wx" });
if ((await readFile(filename, "utf8")) !== before)
throw new Error(`Concurrent edit to ${file}; retry after joining writers`);
await rename(temporary, filename);
} finally {
await rm(temporary, { force: true });
}
}
}
}
if (changed) {
const temporary = `${filename}.${randomUUID()}.tmp`;
try {
await writeFile(temporary, formatQuixosLockDocument(parsed.document), { flag: "wx" });
if ((await readFile(filename, "utf8")) !== before)
throw new Error(`Concurrent edit to ${file}; retry after joining writers`);
await rename(temporary, filename);
} finally {
await rm(temporary, { force: true });
const snapshotStarted = performance.now();
const commit = await snapshotCommit(root);
if (process.env.QUIXOS_TRACE_CAPTURE === "1")
console.error(
`[${new Date().toISOString()}] Capture: ${directory} snapshot: ${Math.round(performance.now() - snapshotStarted)}ms`,
);
phase = "publication";
const ref = retentionTagForCommit(commit);
const remote = await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref]);
if (remote && remote.split(/\s+/)[0] !== commit)
throw new Error(`Conflicting immutable retention ref ${ref}`);
if (!remote) {
await command(root, "git", ["push", node.source.repository, `${commit}:${ref}`]);
if (
(await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref])).split(/\s+/)[0] !==
commit
)
throw new Error("Published source retention was not observed");
}
complete.set(directory, { ...node.source, commit });
} catch (error) {
blockers.push({ directory, phase, message: String(error).slice(0, 4000) });
}
}
const snapshotStarted = performance.now();
const commit = await snapshotCommit(root);
if (process.env.QUIXOS_TRACE_CAPTURE === "1")
console.error(
`[${new Date().toISOString()}] Capture: ${directory} snapshot: ${Math.round(performance.now() - snapshotStarted)}ms`,
);
phase = "publication";
const ref = retentionTagForCommit(commit);
const remote = await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref]);
if (remote && remote.split(/\s+/)[0] !== commit) throw new Error(`Conflicting immutable retention ref ${ref}`);
if (!remote) await command(root, "git", ["push", node.source.repository, `${commit}:${ref}`]);
if ((await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref])).split(/\s+/)[0] !== commit)
throw new Error("Published source retention was not observed");
complete.set(directory, { ...node.source, commit });
} catch (error) {
blockers.push({ directory, phase, message: String(error).slice(0, 4000) });
}
active.delete(directory);
return complete.has(directory);
return complete.has(directory);
});
})();
pending.set(directory, result);
return result;
};
// Include newly created, not-yet-imported resources, then the root.
if (target === "root") for (const directory of [...nodes.keys()].filter((d) => d !== "root")) await visit(directory);
if (target === "root") await Promise.all([...nodes.keys()].filter((d) => d !== "root").map(visit));
await visit(target);
for (let index = blockers.length - 1; index >= 0; index--)
if (!visited.has(blockers[index].directory)) blockers.splice(index, 1);
for (const [directory, source] of complete) {
const recheckStarted = performance.now();
await mapConcurrent([...complete], async ([directory, source]) => {
try {
if ((await snapshotCommit(path.join(context.workbench, directory))) !== source.commit)
throw new Error("Source advanced while converging; join writers and retry");
} catch (error) {
blockers.push({ directory, phase: "concurrent-edit", message: String(error) });
}
}
});
profile("capture-recheck", recheckStarted, { repositories: complete.size });
// Persist successful selections even if another repository is still broken.
// Recovery must not depend on all parents succeeding in the same invocation.
const graph = JSON.parse(graphBefore);
@@ -238,11 +304,14 @@ export async function convergeAuthoring(start: string, target = "root") {
await rm(temporary, { force: true });
}
}
profile("capture", started, { repositories: nodes.size, snapshots: complete.size * 2, target });
return {
workbench: context.workbench,
converged: blockers.length === 0,
candidate: blockers.length ? null : (complete.get(target) ?? null),
retained: [...complete].map(([directory, source]) => ({ directory, source })),
retained: [...complete]
.sort(([left], [right]) => left.localeCompare(right))
.map(([directory, source]) => ({ directory, source })),
worklist: blockers,
verificationEvidence: false,
activated: false,
+72 -45
View File
@@ -13,13 +13,14 @@ export const checkerIdentity = () =>
export async function snapshotCommit(root: string): Promise<string> {
const run = async (...args: string[]) =>
(await execFile("jj", args, { cwd: root, env: environment() })).stdout.trim();
await run("status");
// jj resolve --list exits 1 on a clean revision. Query structured revision
// metadata instead of depending on diagnostic wording or swallowing errors.
if ((await run("--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "conflict")) !== "false")
throw new Error("Resolve source conflicts before verification");
const commit = await run("--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "commit_id");
if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit)) throw new Error("Verification requires an exact jj commit");
// A single normal log snapshots the working copy and observes both values
// from that exact revision. Separate status/log processes add no evidence.
const [conflict, commit] = (await run("log", "--no-graph", "-r", "@", "-T", 'conflict ++ "\\n" ++ commit_id')).split(
"\n",
);
if (conflict !== "false") throw new Error("Resolve source conflicts before verification");
if (!commit || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit))
throw new Error("Verification requires an exact jj commit");
await execFile("git", ["diff", "--exit-code", "--no-ext-diff", "--no-textconv", commit, "--"], {
cwd: root,
env: environment(),
@@ -110,6 +111,7 @@ export async function buildImmutableCandidate(
kind: "workspace" | "interface" | "package" | "build",
logFile: string,
contractOnly = false,
observe?: (phase: "nixEvaluationMs" | "nixBuildMs", ms: number) => void,
): Promise<string> {
if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(source.commit))
throw new Error("An immutable candidate requires an exact commit");
@@ -126,16 +128,52 @@ export async function buildImmutableCandidate(
log.once("open", () => resolve());
log.once("error", reject);
});
return await new Promise((resolve, reject) => {
let output = "",
tail = "",
failure: Error | undefined;
const child = spawn(
"nix",
[
"build",
"--impure",
"--file",
const run = async (executable: string, args: string[]) =>
await new Promise<string>((resolve, reject) => {
let output = "",
tail = "",
failure: Error | undefined;
const child = spawn(executable, args, { env: environment(), stdio: ["ignore", "pipe", "pipe"] });
const failedLog = (error: Error) => {
failure = error;
child.kill();
};
log.on("error", failedLog);
child.stdout.on("data", (chunk) => {
output += chunk;
});
child.stderr.on("data", (chunk) => {
log.write(chunk);
tail = (tail + String(chunk)).slice(-6000);
});
child.on("error", (error) => {
failure = error;
});
child.on("close", (code) => {
log.off("error", failedLog);
if (failure) reject(failure);
else if (code !== 0) reject(new Error(`Candidate Nix check failed (${code}). Full log: ${logFile}\n${tail}`));
else resolve(output.trim());
});
});
const timed = async (phase: "nixEvaluationMs" | "nixBuildMs", action: () => Promise<string>) => {
const started = performance.now();
try {
return await action();
} finally {
observe?.(phase, Math.round(performance.now() - started));
}
};
let gcDirectory: string | undefined;
try {
// Evaluation includes import-from-derivation builds (locks and contracts).
// Build the resulting derivation directly, without evaluating the DAG twice.
// Keep the derivation alive across the two processes, including concurrent
// GC. This temporary root is removed on success and on verification failure.
gcDirectory = await fs.mkdtemp(path.join(path.dirname(logFile), ".gc-root-"));
const gcRoot = path.join(gcDirectory, "candidate");
const derivation = await timed("nixEvaluationMs", async () => {
await run("nix-instantiate", [
builder,
"--argstr",
"repository",
@@ -152,33 +190,22 @@ export async function buildImmutableCandidate(
"--arg",
"contractOnly",
contractOnly ? "true" : "false",
"--no-link",
"--print-out-paths",
"-L",
],
{ env: environment(), stdio: ["ignore", "pipe", "pipe"] },
"--add-root",
gcRoot,
"--indirect",
]);
return await fs.readlink(gcRoot);
});
if (!/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+\.drv$/.test(derivation))
throw new Error("Nix returned an invalid candidate derivation");
const artifact = await timed("nixBuildMs", () =>
run("nix", ["build", derivation + "^*", "--no-link", "--print-out-paths", "-L"]),
);
log.on("error", (error) => {
failure = error;
child.kill();
});
child.stdout.on("data", (chunk) => {
output += chunk;
});
child.stderr.on("data", (chunk) => {
log.write(chunk);
tail = (tail + String(chunk)).slice(-6000);
});
child.on("error", (error) => {
failure = error;
});
child.on("close", (code) =>
log.end(() => {
if (failure) reject(failure);
else if (code !== 0 || !/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+$/.test(output.trim()))
reject(new Error(`Candidate Nix check failed (${code}). Full log: ${logFile}\n${tail}`));
else resolve(output.trim());
}),
);
});
if (!/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+$/.test(artifact))
throw new Error("Nix returned an invalid candidate artifact");
return artifact;
} finally {
await new Promise<void>((resolve) => log.end(resolve));
if (gcDirectory) await fs.rm(gcDirectory, { recursive: true, force: true });
}
}
@@ -0,0 +1,40 @@
import { readFile } from "node:fs/promises";
import type { CheckedResourceResolver } from "./assembly.js";
import type { CapabilityResource } from "./parser.js";
/** Internal compiler inputs produced by checked-candidate.nix, not authoring
* receipts. The Nix composition owns their derivations and exact toolchain.
* Ordinary compiler callers omit this map and run all checks directly. */
export async function loadCheckedResources(filename?: string): Promise<CheckedResourceResolver | undefined> {
if (!filename) return undefined;
const entries = JSON.parse(await readFile(filename, "utf8")) as Array<{
kind: string;
repository: string;
commit: string;
artifact: string;
}>;
const resources = new Map<string, string>();
for (const entry of entries) {
const key = `${entry.kind}\0${entry.repository}\0${entry.commit}`;
if (resources.has(key)) throw new Error("Repeated checked resource input");
if (!/^\/nix\/store\/[^/]+$/.test(entry.artifact))
throw new Error("Checked resources must be immutable Nix outputs");
resources.set(key, entry.artifact);
}
return async (kind, source, lock, dependencies) => {
const artifact = resources.get(`${kind}\0${source.repository}\0${source.commit}`);
if (!artifact) return undefined;
const graph = JSON.parse(await readFile(`${artifact}/graph.json`, "utf8"));
const expected = [...dependencies].map(([binding, node]) => ({ binding, resourceKey: node.key }));
if (
graph.quixos.commit !== lock.quixos.commit ||
graph.quixos.repository !== lock.quixos.repository ||
graph.root.kind !== kind ||
graph.root.source.repository !== source.repository ||
graph.root.source.commit !== source.commit ||
JSON.stringify(graph.root.dependencies) !== JSON.stringify(expected)
)
throw new Error("Checked resource input does not match its locked dependency graph");
return JSON.parse(await readFile(`${artifact}/candidate.json`, "utf8")) as CapabilityResource;
};
}
@@ -80,6 +80,7 @@ export async function executionScaffold(
devShells.default = (import (helpers + "/quixos-package-helpers.nix")).mkCaminoYarnShell { inherit pkgs; };
quixosPackages.checkedArtifacts = context:
let
pkgs = camino.packageSet { inherit nixpkgs system context; };
contract = builtins.fromJSON (builtins.readFile context.buildContract);
package = contract.definition.package;
portable = builtins.filter (entry: entry.execution.kind != "remote") package.exports;
+5
View File
@@ -0,0 +1,5 @@
/** Observations belong in logs, never in content-addressed compiler outputs. */
export const profile = (phase: string, started: number, details: Record<string, unknown> = {}) => {
if (process.env.QUIXOS_TRACE_PROFILE === "1")
console.error(`[qx-profile] ${JSON.stringify({ phase, ms: Math.round(performance.now() - started), ...details })}`);
};
+5 -1
View File
@@ -6,11 +6,12 @@ import { bindingSchema } from "../bindings/index.js";
import { compileCapabilityResourceRepository, type ResolvedCapabilityResource } from "./assembly.js";
import { createGitCapabilityResolver } from "./git-resolver.js";
import { loadQueryGenerator } from "./query-generator.js";
import { loadCheckedResources } from "./checked-resources.js";
const usage = `usage: quixos-resource-compile --root DIRECTORY --kind interface|package|build
--repository URL --commit GIT_REV --checkout-root DIRECTORY
[--snapshot-map PATH] [--snapshot-only true] [--graph-out PATH] [--schema-out PATH]
[--query-generator BUILD_MODULE_PATH]
[--query-generator BUILD_MODULE_PATH] [--checked-resources NIX_MAP]
Resolves a standalone resource's recursive dependency-lock graph, validates
its interface.qx or package.qx manifest, and emits the checked resource as JSON.`;
@@ -33,6 +34,7 @@ const parseArgs = (args: string[]) => {
"--graph-out",
"--schema-out",
"--query-generator",
"--checked-resources",
].includes(key) ||
values.has(key)
)
@@ -69,6 +71,7 @@ const parseArgs = (args: string[]) => {
graphOut: values.get("--graph-out"),
schemaOut: values.get("--schema-out"),
queryGenerator: values.get("--query-generator"),
checkedResources: values.get("--checked-resources"),
} as const;
};
@@ -111,6 +114,7 @@ const main = async () => {
},
resolveResource,
generateQueries: await loadQueryGenerator(options.queryGenerator),
checkedResource: await loadCheckedResources(options.checkedResources),
});
if (options.graphOut) {
await writeFile(
+11 -1
View File
@@ -5,6 +5,8 @@ import process from "node:process";
import { compileWorkspaceRepository } from "./assembly.js";
import { createGitCapabilityResolver } from "./git-resolver.js";
import { loadQueryGenerator } from "./query-generator.js";
import { loadCheckedResources } from "./checked-resources.js";
import { profile } from "./profile.js";
import { compileExecutionPlan, packageBuildContexts } from "../execution-world/index.js";
import { bindingSchema, specializeBindingSchema } from "../bindings/index.js";
import {
@@ -18,7 +20,7 @@ const usage = `usage: quixos-workspace-compile --root DIRECTORY --checkout-root
[--snapshot-map PATH] [--graph-out PATH] [--workspace-id ID] [--workspace-revision-id ID]
[--source-root-commit GIT_REV] [--baseline PLAN_JSON] [--evolution-out PATH]
[--reviews REVIEW_JSON] [--schemas-out PATH] [--execution-plan-out PATH] [--build-contexts-out PATH]
[--query-generator BUILD_MODULE_PATH]
[--query-generator BUILD_MODULE_PATH] [--checked-resources NIX_MAP]
Resolves a workspace's recursive resource-lock graph, clones every exact
resource revision, validates standalone interface/package manifests, and emits
@@ -50,6 +52,7 @@ const parseArgs = (args: string[]) => {
evolutionOut: values.get("--evolution-out"),
reviews: values.get("--reviews"),
queryGenerator: values.get("--query-generator"),
checkedResources: values.get("--checked-resources"),
};
};
@@ -63,6 +66,7 @@ const main = async () => {
checkoutRoot: options.checkoutRoot,
snapshotMap: options.snapshotMap,
});
const assemblyStarted = performance.now();
const assembled = await compileWorkspaceRepository({
rootDirectory: options.rootDirectory,
workspaceId: options.workspaceId,
@@ -70,7 +74,9 @@ const main = async () => {
sourceRootCommit: options.sourceRootCommit,
resolveResource,
generateQueries: await loadQueryGenerator(options.queryGenerator),
checkedResource: await loadCheckedResources(options.checkedResources),
});
profile("workspace-assembly", assemblyStarted, { resources: assembled.resources.length });
if (options.graphOut) {
await writeFile(
@@ -110,12 +116,15 @@ const main = async () => {
}
const candidate = { ...assembled.workspace, executionContracts: runtimeContracts(assembled.workspace) };
if (options.executionPlanOut || options.buildContextsOut) {
const executionStarted = performance.now();
const plan = compileExecutionPlan(assembled.workspace);
if (options.executionPlanOut) await writeFile(options.executionPlanOut, `${JSON.stringify(plan, null, 2)}\n`);
if (options.buildContextsOut)
await writeFile(options.buildContextsOut, JSON.stringify(packageBuildContexts(plan.definition.bindingSchema)));
profile("execution-plan-and-contexts", executionStarted);
}
if (options.schemasOut) {
const schemasStarted = performance.now();
const schemas: Record<string, unknown> = {};
for (const node of assembled.capabilityResources.filter((entry) => entry.kind === "package")) {
const closure = new Map<string, typeof node>();
@@ -133,6 +142,7 @@ const main = async () => {
);
}
await writeFile(options.schemasOut, JSON.stringify(schemas));
profile("package-schemas", schemasStarted);
}
if (options.evolutionOut) {
const baseline = options.baseline
+63 -5
View File
@@ -107,6 +107,35 @@ package Runtime id "package:runtime" revision "package:runtime@1" {
assert.equal(resource.resource.kind, "package");
assert.equal(resource.resources.length, 2);
assert.equal(resource.directResources.size, 1);
const checked = new Map(result.resources.map((node) => [node.key, node.resource]));
let generated = 0;
const reused = await compileWorkspaceRepository({
rootDirectory: root,
resolveResource: async (source, kind) => ({
directory: directories.get(`${kind}\0${source.repository}\0${source.commit}`)!,
}),
checkedResource: async (kind, source) => {
const value = checked.get(`${kind}\0${source.repository}\0${source.commit}`);
return value?.kind === "build" ? undefined : value;
},
generateQueries: async () => {
generated++;
return { queries: [], documents: new Map() };
},
});
assert.deepEqual(reused, result, "Nix-composed resource checks preserve full assembly results");
assert.equal(generated, 0, "unchanged checked packages do not run their query frontend again");
await assert.rejects(
compileWorkspaceRepository({
rootDirectory: root,
resolveResource: async (source, kind) => ({
directory: directories.get(`${kind}\0${source.repository}\0${source.commit}`)!,
}),
checkedResource: async () => (resource.resource.kind === "build" ? undefined : resource.resource),
}),
/Checked resource identity mismatch/,
);
});
test("resource repositories cannot own workspace Quixos selection policy", async (context) => {
@@ -249,19 +278,48 @@ test("resource-owned fragment imports compile through the locked package closure
}`,
);
await writeFile(path.join(dirs.screen!, "screen.graphql"), "query Screen {root {...Row}}");
const result = await compileCapabilityResourceRepository({
const options = {
rootDirectory: dirs.screen!,
kind: "package",
source: { resolver: "git", repository: "https://example.test/screen.git", commit: packageCommit },
resolveResource: async (source) => {
kind: "package" as const,
source: { resolver: "git" as const, repository: "https://example.test/screen.git", commit: packageCommit },
resolveResource: async (source: { repository: string }) => {
const name = new URL(source.repository).pathname.slice(1, -4);
assert.ok(dirs[name], name);
return { directory: dirs[name]! };
},
});
};
const result = await compileCapabilityResourceRepository(options);
assert.equal(result.resource.kind, "package");
if (result.resource.kind !== "package") throw Error("expected package");
const view = result.resource.revision.checkedQueries!.find((q) => q.declaration.id === "view")!;
assert.equal(view.selection[0]!.selection[0]!.fragmentSpread?.package, "rows@1");
assert.equal(result.resources.length, 3);
const checked = new Map(result.resources.map((node) => [node.key, node.resource]));
let generated = 0;
const reused = await compileCapabilityResourceRepository({
...options,
checkedResource: async (kind, source) => {
if (source.repository === options.source.repository) return undefined;
const resource = checked.get(`${kind}\0${source.repository}\0${source.commit}`);
return resource?.kind === "build" ? undefined : resource;
},
generateQueries: async () => {
generated++;
return { queries: [], documents: new Map() };
},
});
assert.deepEqual(reused, result, "cached imported fragments and full checking produce identical contracts");
assert.equal(generated, 1, "only the uncached package checks its queries");
await writeFile(path.join(dirs.screen!, "screen.graphql"), "query Screen {root {unknownField}}");
await assert.rejects(
compileCapabilityResourceRepository({
...options,
source: { ...options.source, commit: "5".repeat(40) },
checkedResource: async (kind, source) => {
const resource = checked.get(`${kind}\0${source.repository}\0${source.commit}`);
return resource?.kind === "build" ? undefined : resource;
},
}),
/unknownField/,
);
});