Instrument and improve incremental workspace checks

Profile source capture, Nix evaluation/realization, resource/query checking,
and implementation builds without putting timings into immutable outputs.
Keep modular repositories; bound independent capture work, snapshot jj once
per observation, and retain final concurrent-edit and Git completeness checks.

Compose exact resource/closure/frontend/toolchain checks as memoized Nix
derivations. Preserve whole-workspace validation and the full compiler path
when no checked dependency is supplied. Share canonical Nixpkgs by content,
index workspace build contexts once, and narrow lock/driver/compiler inputs
with conservative complete-source fallback. Batch React witness checks only
when distinct source closures cannot share ambient declarations; otherwise
run the original independent TypeScript checks.

Seed local authoring toolchains with exact canonical fetch identities when
they match the selected inputs; retain store-path inputs for overrides.
Add generic correctness checks rather than permanent Reader timing/content
fixtures. Verify actual Reader edits and rejection/recovery in a private
workbench, preserving the original user's workbench and runtime.

Query checking currently caches at resource/closure boundaries, not individual
query semantic dependency boundaries. Template helper pins and reproducible
fixture snapshots follow in a separate publication change.
This commit is contained in:
Timothy J. Aveni
2026-10-01 04:03:19 -07:00
parent e8d19d2bb7
commit 7c1830bcd5
15 changed files with 527 additions and 141 deletions
+3 -1
View File
@@ -81,11 +81,13 @@
install -Dm644 nix/checked-package.nix "$out/share/checked-package.nix" install -Dm644 nix/checked-package.nix "$out/share/checked-package.nix"
install -Dm644 nix/checked-artifacts.nix "$out/share/checked-artifacts.nix" install -Dm644 nix/checked-artifacts.nix "$out/share/checked-artifacts.nix"
install -Dm644 nix/package-context.nix "$out/share/package-context.nix" install -Dm644 nix/package-context.nix "$out/share/package-context.nix"
install -Dm644 nix/lock-source.nix "$out/share/lock-source.nix"
install -Dm644 nix/package-context.mjs "$out/share/package-context.mjs" install -Dm644 nix/package-context.mjs "$out/share/package-context.mjs"
install -Dm644 nix/check-package.mjs "$out/share/check-package.mjs" install -Dm644 nix/check-package.mjs "$out/share/check-package.mjs"
install -Dm644 nix/check-execution.mjs "$out/share/check-execution.mjs" install -Dm644 nix/check-execution.mjs "$out/share/check-execution.mjs"
substitute nix/checked-candidate.nix "$out/share/checked-candidate.nix" \ substitute nix/checked-candidate.nix "$out/share/checked-candidate.nix" \
--replace-fail '@nixpkgs@' '${pkgs.path}' --replace-fail '@nixpkgs@' '${pkgs.path}' \
--replace-fail '@nixpkgsNarHash@' '${nixpkgs.narHash}'
cp --reflink=auto --recursive grammar "$out/grammar" cp --reflink=auto --recursive grammar "$out/grammar"
cp --reflink=auto --recursive proto "$out/proto" cp --reflink=auto --recursive proto "$out/proto"
cp --reflink=auto --recursive dist "$out/dist" cp --reflink=auto --recursive dist "$out/dist"
+62 -12
View File
@@ -9,7 +9,11 @@
system ? builtins.currentSystem, system ? builtins.currentSystem,
}: }:
let let
pkgs = import (/. + "@nixpkgs@") { inherit system; }; pkgs = import (/. + "@nixpkgs@") {
inherit system;
config = { };
overlays = [ ];
};
protocol = builtins.storePath generator; protocol = builtins.storePath generator;
sourceKey = source: "${source.kind}:${source.repository}@${source.commit}"; sourceKey = source: "${source.kind}:${source.repository}@${source.commit}";
fetch = fetch =
@@ -26,8 +30,9 @@ let
source: source:
let let
directory = fetch source; directory = fetch source;
lockSource = import ./lock-source.nix { source = directory; };
lockFile = pkgs.runCommand "qx-source-lock.json" { } '' lockFile = pkgs.runCommand "qx-source-lock.json" { } ''
${protocol}/bin/quixos-lock-check ${directory}/quixos.lock > "$out" ${protocol}/bin/quixos-lock-check ${lockSource}/quixos.lock > "$out"
''; '';
lock = builtins.fromJSON (builtins.readFile lockFile); lock = builtins.fromJSON (builtins.readFile lockFile);
in in
@@ -124,15 +129,40 @@ let
}) (builtins.filter (entry: entry.kind != "workspace") (resourceClosure true node)); }) (builtins.filter (entry: entry.kind != "workspace") (resourceClosure true node));
} }
); );
compile = # Each resource is checked once per exact source/closure/frontend/toolchain.
# Composition imports these compiler outputs; authoring receipts are never
# build inputs. Missing entries in ordinary CLI use take the full check path.
compiledNodes = builtins.mapAttrs (_: node: compileNode node) nodeTable;
compile = node: compiledNodes.${sourceKey node};
checkedResources =
node: node:
pkgs.runCommand "qx-${node.kind}-contract" { } '' pkgs.writeText "qx-checked-resources.json" (
builtins.toJSON (
map
(dependency: {
inherit (dependency) kind repository commit;
artifact = compile dependency;
})
(
builtins.filter (
dependency:
dependency.kind != "workspace"
&& dependency.kind != "build"
&& sourceKey dependency != sourceKey node
) (resourceClosure true node)
)
)
);
compileNode =
node:
pkgs.runCommand "qx-${node.kind}-contract" { QUIXOS_TRACE_PROFILE = "1"; } ''
mkdir -p "$out" mkdir -p "$out"
${ ${
if node.kind == "workspace" then if node.kind == "workspace" then
'' ''
${protocol}/bin/quixos-workspace-compile --root ${node.directory} \ ${protocol}/bin/quixos-workspace-compile --root ${node.directory} \
${queryGeneratorFlags node} \ ${queryGeneratorFlags root} \
--checked-resources ${checkedResources node} \
--source-root-commit ${pkgs.lib.escapeShellArg node.commit} \ --source-root-commit ${pkgs.lib.escapeShellArg node.commit} \
--checkout-root "$TMPDIR/checkouts" --snapshot-map ${snapshots node} \ --checkout-root "$TMPDIR/checkouts" --snapshot-map ${snapshots node} \
--graph-out "$out/graph.json" --schemas-out "$out/package-schemas.json" \ --graph-out "$out/graph.json" --schemas-out "$out/package-schemas.json" \
@@ -144,7 +174,8 @@ let
else else
'' ''
${protocol}/bin/quixos-resource-compile --root ${node.directory} \ ${protocol}/bin/quixos-resource-compile --root ${node.directory} \
${queryGeneratorFlags node} \ ${queryGeneratorFlags root} \
--checked-resources ${checkedResources node} \
--kind ${node.kind} --repository ${pkgs.lib.escapeShellArg node.repository} \ --kind ${node.kind} --repository ${pkgs.lib.escapeShellArg node.repository} \
--commit ${pkgs.lib.escapeShellArg node.commit} \ --commit ${pkgs.lib.escapeShellArg node.commit} \
--checkout-root "$TMPDIR/checkouts" --snapshot-map ${snapshots node} --snapshot-only true \ --checkout-root "$TMPDIR/checkouts" --snapshot-map ${snapshots node} --snapshot-only true \
@@ -153,6 +184,18 @@ let
} }
''; '';
contract = compile root; contract = compile root;
# Parse shared workspace outputs once. Repeating fromJSON in every package
# makes composition quadratic in package count and in the serialized world.
workspacePackages = builtins.listToAttrs (
map
(pkg: {
name = sourceKey (pkg.source // { kind = "package"; });
value = pkg;
})
(builtins.fromJSON (builtins.readFile "${contract}/execution-plan.json"))
.definition.bindingSchema.packages
);
workspaceContexts = builtins.fromJSON (builtins.readFile "${contract}/build-contexts.json");
buildDependencies = buildDependencies =
node: node:
builtins.listToAttrs ( builtins.listToAttrs (
@@ -218,17 +261,19 @@ let
if execution && kind == "workspace" then if execution && kind == "workspace" then
{ {
revision = revision =
pkgs.lib.findFirst workspacePackages.${sourceKey node}
(pkg: pkg.source.repository == node.repository && pkg.source.commit == node.commit) or (throw "Source package missing from workspace compilation: ${node.repository}");
(throw "Source package missing from workspace compilation: ${node.repository}")
(builtins.fromJSON (builtins.readFile "${contract}/execution-plan.json"))
.definition.bindingSchema.packages;
} }
else else
builtins.fromJSON (builtins.readFile "${compiled}/candidate.json"); builtins.fromJSON (builtins.readFile "${compiled}/candidate.json");
buildContext = import ./package-context.nix { buildContext = import ./package-context.nix {
inherit pkgs protocol; inherit pkgs protocol;
contexts = if kind == "workspace" then "${contract}/build-contexts.json" else null; contexts = if kind == "workspace" then "${contract}/build-contexts.json" else null;
contextData =
if execution && kind == "workspace" then
workspaceContexts.${candidate.revision.revisionId}
else
null;
plan = "${compiled}/bindings.json"; plan = "${compiled}/bindings.json";
packageRevisionId = candidate.revision.revisionId; packageRevisionId = candidate.revision.revisionId;
}; };
@@ -254,6 +299,11 @@ let
typescript = runtimePackages.camino-replica-core.dependencies; typescript = runtimePackages.camino-replica-core.dependencies;
packageRevisionId = candidate.revision.revisionId; packageRevisionId = candidate.revision.revisionId;
dependencies = buildDependencies node; dependencies = buildDependencies node;
buildEnvironment = {
inherit pkgs system;
nixpkgsSource = toString pkgs.path;
narHash = "@nixpkgsNarHash@";
};
} }
// pkgs.lib.optionalAttrs (componentEntries != [ ]) { // pkgs.lib.optionalAttrs (componentEntries != [ ]) {
react = { react = {
@@ -310,7 +360,7 @@ in
pkgs.runCommand (if contractOnly then "qx-contract" else "qx-checked-candidate") { } '' pkgs.runCommand (if contractOnly then "qx-contract" else "qx-checked-candidate") { } ''
mkdir -p "$out" mkdir -p "$out"
cp ${ cp ${
pkgs.writeText "qx-candidate-source.json" (builtins.toJSON { inherit repository commit kind; }) builtins.toFile "qx-candidate-source.json" (builtins.toJSON { inherit repository commit kind; })
} "$out/source.json" } "$out/source.json"
cp ${contract}/candidate.json "$out/candidate.json" cp ${contract}/candidate.json "$out/candidate.json"
cp ${contract}/graph.json "$out/graph.json" cp ${contract}/graph.json "$out/graph.json"
+16
View File
@@ -0,0 +1,16 @@
# Lock parsing only reads this one file when no import token can occur. A
# conservative substring probe also sends comments/URLs containing "import"
# through the complete-source path. Fragment names may have any extension.
{ source }:
let
regular = ((builtins.readDir source)."quixos.lock" or null) == "regular";
text = if regular then builtins.readFile (source + "/quixos.lock") else "import";
in
if builtins.replaceStrings [ "import" ] [ "" ] text != text then
source
else
builtins.path {
path = source;
name = "qx-lock-source";
filter = filename: _: filename == toString source || filename == "${source}/quixos.lock";
}
+42
View File
@@ -0,0 +1,42 @@
{ pkgs, protocol }:
let
base = ''quixos-lock version 1 { quixos source { repository "https://example.test/quixos.git"; commit "1111111111111111111111111111111111111111"; } }'';
fixture =
extra: lock:
pkgs.runCommand "qx-lock-source-fixture" { } ''
mkdir -p "$out/nested"
printf '%s' ${pkgs.lib.escapeShellArg lock} > "$out/quixos.lock"
printf '%s' ${pkgs.lib.escapeShellArg extra} > "$out/implementation.ts"
printf '%s' 'quixos-lock fragment version 1 {}' > "$out/nested/dependencies.txt"
'';
first = fixture "first implementation" base;
edited = fixture "edited implementation" base;
changed = fixture "first implementation" (
builtins.replaceStrings
[ "1111111111111111111111111111111111111111" ]
[ "2222222222222222222222222222222222222222" ]
base
);
fragments = fixture "first implementation" (
builtins.replaceStrings [ "quixos source" ] [ ''import "nested/dependencies.txt"; quixos source'' ]
base
);
conservative = fixture "first implementation" (base + "\n// import in a comment\n");
slice = source: import ./lock-source.nix { inherit source; };
parse =
source:
pkgs.runCommand "qx-source-lock.json" { } ''
${protocol}/bin/quixos-lock-check ${slice source}/quixos.lock > "$out"
'';
in
assert slice first == slice edited;
assert (parse first).drvPath == (parse edited).drvPath;
assert slice first != slice changed;
pkgs.runCommand "qx-lock-source-check" { } ''
${protocol}/bin/quixos-lock-check ${slice first}/quixos.lock > first.json
${protocol}/bin/quixos-lock-check ${slice edited}/quixos.lock > edited.json
cmp first.json edited.json
${protocol}/bin/quixos-lock-check ${slice conservative}/quixos.lock > comment.json
cmp first.json comment.json
${protocol}/bin/quixos-lock-check ${slice fragments}/quixos.lock > "$out"
''
+4 -1
View File
@@ -5,6 +5,7 @@
protocol, protocol,
plan ? null, plan ? null,
contexts ? null, contexts ? null,
contextData ? null,
packageRevisionId, packageRevisionId,
}: }:
let let
@@ -14,7 +15,9 @@ let
${pkgs.lib.escapeShellArg packageRevisionId} > "$out" ${pkgs.lib.escapeShellArg packageRevisionId} > "$out"
''; '';
context = context =
if contexts == null then if contextData != null then
contextData
else if contexts == null then
builtins.fromJSON (builtins.readFile projected) builtins.fromJSON (builtins.readFile projected)
else else
(builtins.fromJSON (builtins.readFile contexts)).${packageRevisionId}; (builtins.fromJSON (builtins.readFile contexts)).${packageRevisionId};
+57 -1
View File
@@ -1,5 +1,6 @@
import { readFile, realpath } from "node:fs/promises"; import { readFile, realpath } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import { profile } from "./profile.js";
import { loadQxSources, resolveQxSources } from "./source-loader.js"; import { loadQxSources, resolveQxSources } from "./source-loader.js";
import { compileQuery, compileRepositoryQuery } from "../query/compile.js"; import { compileQuery, compileRepositoryQuery } from "../query/compile.js";
import type { QueryDeclaration } from "../query/types.js"; import type { QueryDeclaration } from "../query/types.js";
@@ -43,6 +44,15 @@ export type CapabilityRepositoryResolver = (
kind: LockedResource["kind"], kind: LockedResource["kind"],
) => Promise<CapabilityRepositorySnapshot>; ) => Promise<CapabilityRepositorySnapshot>;
/** Already checked outputs of this compiler, composed by the Nix build DAG.
* Source capture/check reports never grant this authority. */
export type CheckedResourceResolver = (
kind: LockedResource["kind"],
source: GitSource,
lock: QuixosRepositoryLock,
dependencies: ReadonlyMap<string, ResolvedCapabilityResource>,
) => Promise<CapabilityResource | undefined>;
/** An explicitly selected build frontend supplies ordinary query sources. The /** An explicitly selected build frontend supplies ordinary query sources. The
* assembler never chooses a frontend by inspecting an interface/contract ID. * assembler never chooses a frontend by inspecting an interface/contract ID.
* Generated sources go through the same query and imported-fragment checks. */ * Generated sources go through the same query and imported-fragment checks. */
@@ -221,7 +231,12 @@ const createResourceGraphResolver = (
quixosCommit: string, quixosCommit: string,
resolveResource: CapabilityRepositoryResolver, resolveResource: CapabilityRepositoryResolver,
generateQueries?: PackageQueryGenerator, generateQueries?: PackageQueryGenerator,
checkedResource?: CheckedResourceResolver,
) => { ) => {
const graphStarted = performance.now();
let checkedResources = 0,
reusedResources = 0,
checkedQueries = 0;
const resolved = new Map<string, Promise<ResolvedCapabilityResource>>(); const resolved = new Map<string, Promise<ResolvedCapabilityResource>>();
const active: string[] = []; const active: string[] = [];
@@ -275,6 +290,30 @@ const createResourceGraphResolver = (
}; };
} }
const dependencyClosure = resourceClosure(directPairs.map(([, node]) => node)); const dependencyClosure = resourceClosure(directPairs.map(([, node]) => node));
const dependencies = new Map(
directPairs.map(([dependency, node]) => [bindingKey(dependency.kind, dependency.binding), node]),
);
const checked = await checkedResource?.(locked.kind, locked.source, lockResult.lock, dependencies);
if (checked) {
if (
checked.kind !== locked.kind ||
checked.revision.source.repository !== locked.source.repository ||
checked.revision.source.commit !== locked.source.commit
)
throw new Error("Checked resource identity mismatch");
assertImportsMatchLock("Checked resource", checked.imports, lockResult.lock.resources);
reusedResources++;
return {
key,
kind: locked.kind,
source: locked.source,
directory: snapshot.directory,
lock: lockResult.lock,
resource: structuredClone(checked),
dependencies,
};
}
const checkStarted = performance.now();
const environment = environmentFor(directPairs, dependencyClosure); const environment = environmentFor(directPairs, dependencyClosure);
const manifestName = locked.kind === "interface" ? "interface.qx" : "package.qx"; const manifestName = locked.kind === "interface" ? "interface.qx" : "package.qx";
const manifestPath = path.join(snapshot.directory, manifestName); const manifestPath = path.join(snapshot.directory, manifestName);
@@ -399,6 +438,15 @@ const createResourceGraphResolver = (
compiled.resource.revision.migrationCatalog = catalog; compiled.resource.revision.migrationCatalog = catalog;
} }
} }
checkedResources++;
const queries =
compiled.resource.kind === "package" ? (compiled.resource.revision.checkedQueries?.length ?? 0) : 0;
checkedQueries += queries;
profile("resource-check", checkStarted, {
kind: locked.kind,
resource: compiled.resource.revision.displayName,
queries,
});
return { return {
key, key,
kind: locked.kind, kind: locked.kind,
@@ -420,7 +468,11 @@ const createResourceGraphResolver = (
return { return {
visit, visit,
nodes: async () => await Promise.all(resolved.values()), nodes: async () => {
const nodes = await Promise.all(resolved.values());
profile("resource-graph", graphStarted, { checkedResources, reusedResources, checkedQueries });
return nodes;
},
}; };
}; };
@@ -430,6 +482,7 @@ export const compileCapabilityResourceRepository = async (options: {
source: GitSource; source: GitSource;
resolveResource: CapabilityRepositoryResolver; resolveResource: CapabilityRepositoryResolver;
generateQueries?: PackageQueryGenerator; generateQueries?: PackageQueryGenerator;
checkedResource?: CheckedResourceResolver;
}): Promise<CompiledCapabilityResourceRepository> => { }): Promise<CompiledCapabilityResourceRepository> => {
const lockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock")); const lockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock"));
if (!lockResult.ok) { if (!lockResult.ok) {
@@ -439,6 +492,7 @@ export const compileCapabilityResourceRepository = async (options: {
lockResult.lock.quixos.commit, lockResult.lock.quixos.commit,
options.resolveResource, options.resolveResource,
options.generateQueries, options.generateQueries,
options.checkedResource,
); );
const root = await resolver.visit( const root = await resolver.visit(
{ {
@@ -466,6 +520,7 @@ export const compileWorkspaceRepository = async (options: {
/** An editor's proposed source snapshot; locks and dependency revisions remain exact. */ /** An editor's proposed source snapshot; locks and dependency revisions remain exact. */
readSource?: (name: string) => Promise<string>; readSource?: (name: string) => Promise<string>;
generateQueries?: PackageQueryGenerator; generateQueries?: PackageQueryGenerator;
checkedResource?: CheckedResourceResolver;
}): Promise<CompiledWorkspaceRepository> => { }): Promise<CompiledWorkspaceRepository> => {
const rootLockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock")); const rootLockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock"));
if (!rootLockResult.ok) { if (!rootLockResult.ok) {
@@ -476,6 +531,7 @@ export const compileWorkspaceRepository = async (options: {
rootLock.quixos.commit, rootLock.quixos.commit,
options.resolveResource, options.resolveResource,
options.generateQueries, options.generateQueries,
options.checkedResource,
); );
const directPairs: Array<[LockedResource, ResolvedCapabilityResource]> = []; const directPairs: Array<[LockedResource, ResolvedCapabilityResource]> = [];
+4 -1
View File
@@ -64,7 +64,7 @@ export async function checkAuthoring(
// Repository-scoped agents can check separate immutable candidates in parallel. // Repository-scoped agents can check separate immutable candidates in parallel.
const capture = promisify(callback)("quixos-qx", ["converge", context.workbench, directory], { const capture = promisify(callback)("quixos-qx", ["converge", context.workbench, directory], {
maxBuffer: 4 * 1024 * 1024, maxBuffer: 4 * 1024 * 1024,
env: { ...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_TRACE_CAPTURE: "1" }, env: { ...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_TRACE_CAPTURE: "1", QUIXOS_TRACE_PROFILE: "1" },
}); });
console.error(`Capture details: ${path.join(output, "capture.log")}`); console.error(`Capture details: ${path.join(output, "capture.log")}`);
capture.child.stderr?.on("data", (chunk) => appendFileSync(path.join(output, "capture.log"), chunk)); capture.child.stderr?.on("data", (chunk) => appendFileSync(path.join(output, "capture.log"), chunk));
@@ -94,6 +94,9 @@ export async function checkAuthoring(
resource.kind, resource.kind,
path.join(output, "nix.log"), path.join(output, "nix.log"),
options.contractOnly, options.contractOnly,
(phase, ms) => {
timings[phase] = ms;
},
); );
} finally { } finally {
timings.immutableCheckMs = Math.round(performance.now() - buildStarted); timings.immutableCheckMs = Math.round(performance.now() - buildStarted);
+142 -73
View File
@@ -5,6 +5,7 @@ import { promisify } from "node:util";
import { randomUUID } from "node:crypto"; import { randomUUID } from "node:crypto";
import { authoringContext } from "./authoring-context.js"; import { authoringContext } from "./authoring-context.js";
import { snapshotCommit } from "./checked-build.js"; import { snapshotCommit } from "./checked-build.js";
import { profile } from "./profile.js";
import { import {
loadQuixosLock, loadQuixosLock,
parseQuixosLockDocument, parseQuixosLockDocument,
@@ -14,6 +15,42 @@ import {
} from "../resource-lock/index.js"; } from "../resource-lock/index.js";
const execFile = promisify(callback); const execFile = promisify(callback);
// Bound independent repository operations. One visit owns each repository;
// all visits finish before the final concurrent-edit validation pass.
const concurrency = 4;
async function mapConcurrent<T, R>(values: readonly T[], action: (value: T) => Promise<R>): Promise<R[]> {
const results = new Array<R>(values.length);
let next = 0;
await Promise.all(
Array.from({ length: Math.min(concurrency, values.length) }, async () => {
while (next < values.length) {
const index = next++;
results[index] = await action(values[index]);
}
}),
);
return results;
}
const limiter = () => {
let running = 0;
const waiting: Array<() => void> = [];
return async <T>(action: () => Promise<T>): Promise<T> => {
await new Promise<void>((resolve) => {
if (running < concurrency) {
running++;
resolve();
} else waiting.push(resolve);
});
try {
return await action();
} finally {
const next = waiting.shift();
if (next) next();
else running--;
}
};
};
const command = async (cwd: string, executable: string, args: string[]) => { const command = async (cwd: string, executable: string, args: string[]) => {
const start = performance.now(); const start = performance.now();
// Do not log arguments: transports may contain credentials. Source identities // Do not log arguments: transports may contain credentials. Source identities
@@ -44,13 +81,14 @@ export type AuthoringBlocker = {
* worklist grants typechecking, semantic review or activation approval. * worklist grants typechecking, semantic review or activation approval.
* Caller serializes coordinators; package authors may still be editing. */ * Caller serializes coordinators; package authors may still be editing. */
export async function convergeAuthoring(start: string, target = "root") { export async function convergeAuthoring(start: string, target = "root") {
const started = performance.now();
const context = await authoringContext(start); const context = await authoringContext(start);
const graphFile = path.join(context.workbench, ".quixos/resource-graph.json"); const graphFile = path.join(context.workbench, ".quixos/resource-graph.json");
const graphBefore = await readFile(graphFile, "utf8"); const graphBefore = await readFile(graphFile, "utf8");
const blockers: AuthoringBlocker[] = []; const blockers: AuthoringBlocker[] = [];
const nodes = new Map<string, { directory: string; kind: string; source: GitSource; dependencies: string[] }>(); const nodes = new Map<string, { directory: string; kind: string; source: GitSource; dependencies: string[] }>();
const selected = new Map<string, string>(); const selected = new Map<string, string>();
for (const entry of context.resources) { const inventory = await mapConcurrent(context.resources, async (entry) => {
const root = path.join(context.workbench, entry.directory); const root = path.join(context.workbench, entry.directory);
let repository = entry.source?.repository; let repository = entry.source?.repository;
try { try {
@@ -64,6 +102,9 @@ export async function convergeAuthoring(start: string, target = "root") {
blockers.push({ directory: entry.directory, phase: "source", message: String(error).slice(0, 2000) }); blockers.push({ directory: entry.directory, phase: "source", message: String(error).slice(0, 2000) });
if (!repository) throw error; // The root has no separate registered source. if (!repository) throw error; // The root has no separate registered source.
} }
return { ...entry, repository };
});
for (const { repository, ...entry } of inventory) {
const key = identity(entry.kind, repository); const key = identity(entry.kind, repository);
if (selected.has(key)) throw new Error(`More than one editable checkout for ${repository}`); if (selected.has(key)) throw new Error(`More than one editable checkout for ${repository}`);
selected.set(key, entry.directory); selected.set(key, entry.directory);
@@ -73,7 +114,7 @@ export async function convergeAuthoring(start: string, target = "root") {
dependencies: [], dependencies: [],
}); });
} }
for (const node of nodes.values()) { await mapConcurrent([...nodes.values()], async (node) => {
try { try {
const lock = await loadQuixosLock(path.join(context.workbench, node.directory, "quixos.lock")); const lock = await loadQuixosLock(path.join(context.workbench, node.directory, "quixos.lock"));
if (!lock.ok) throw new Error(lock.diagnostics.map((d) => `${d.fileName}: ${d.message}`).join("\n")); if (!lock.ok) throw new Error(lock.diagnostics.map((d) => `${d.fileName}: ${d.message}`).join("\n"));
@@ -88,97 +129,122 @@ export async function convergeAuthoring(start: string, target = "root") {
} catch (error) { } catch (error) {
blockers.push({ directory: node.directory, phase: "resolution", message: String(error) }); blockers.push({ directory: node.directory, phase: "resolution", message: String(error) });
} }
} });
const complete = new Map<string, GitSource>(), // Detect cycles before starting concurrent futures: two roots awaiting each
active = new Set<string>(); // other must fail clearly rather than deadlocking on their cached promises.
const visited = new Set<string>(); const scanned = new Set<string>();
if (!nodes.has(target)) throw new Error(`Not a registered repository: ${target}`); const detectCycles = (directory: string, parents: string[] = []) => {
const visit = async (directory: string): Promise<boolean> => { if (parents.includes(directory)) {
visited.add(directory);
if (complete.has(directory)) return true;
if (blockers.some((entry) => entry.directory === directory)) return false;
if (active.has(directory)) {
blockers.push({ blockers.push({
directory, directory,
phase: "dependency", phase: "dependency",
message: `Source dependency cycle: ${[...active, directory].join(" -> ")}`, message: `Source dependency cycle: ${[...parents, directory].join(" -> ")}`,
}); });
return false; return;
} }
active.add(directory); if (scanned.has(directory)) return;
const node = nodes.get(directory)!; for (const dependency of nodes.get(directory)!.dependencies) detectCycles(dependency, [...parents, directory]);
for (const dependency of node.dependencies) scanned.add(directory);
if (!(await visit(dependency))) { };
blockers.push({ directory, phase: "dependency", message: `Waiting for ${dependency}` }); for (const directory of nodes.keys()) detectCycles(directory);
active.delete(directory); const complete = new Map<string, GitSource>();
const pending = new Map<string, Promise<boolean>>();
const capture = limiter();
const visited = new Set<string>();
if (!nodes.has(target)) throw new Error(`Not a registered repository: ${target}`);
const visit = (directory: string): Promise<boolean> => {
visited.add(directory);
const existing = pending.get(directory);
if (existing) return existing;
const result = (async () => {
if (blockers.some((entry) => entry.directory === directory)) return false;
const node = nodes.get(directory)!;
const ready = await Promise.all(node.dependencies.map(visit));
const unavailable = node.dependencies.find((_, index) => !ready[index]);
if (unavailable) {
blockers.push({ directory, phase: "dependency", message: `Waiting for ${unavailable}` });
return false; return false;
} }
const root = path.join(context.workbench, directory); return await capture(async () => {
let phase: AuthoringBlocker["phase"] = "source"; const root = path.join(context.workbench, directory);
try { let phase: AuthoringBlocker["phase"] = "source";
const lock = await loadQuixosLock(path.join(root, "quixos.lock")); try {
if (!lock.ok) throw new Error("Lock changed during convergence; retry after joining writers"); const lock = await loadQuixosLock(path.join(root, "quixos.lock"));
for (const file of lock.lock.sourceFiles ?? ["quixos.lock"]) { if (!lock.ok) throw new Error("Lock changed during convergence; retry after joining writers");
const filename = path.join(root, file), for (const file of lock.lock.sourceFiles ?? ["quixos.lock"]) {
before = await readFile(filename, "utf8"); const filename = path.join(root, file),
const parsed = parseQuixosLockDocument(before, file); before = await readFile(filename, "utf8");
if (!parsed.ok) throw new Error(`Invalid lock ${file}`); const parsed = parseQuixosLockDocument(before, file);
let changed = false; if (!parsed.ok) throw new Error(`Invalid lock ${file}`);
for (const dependency of parsed.document.resources) { let changed = false;
const target = selected.get(identity(dependency.kind, dependency.source.repository)); for (const dependency of parsed.document.resources) {
const source = target ? complete.get(target) : undefined; const target = selected.get(identity(dependency.kind, dependency.source.repository));
if (target && !source) const source = target ? complete.get(target) : undefined;
throw new Error(`Dependencies changed during convergence (${dependency.binding}); join writers and retry`); if (target && !source)
if (source && source.commit !== dependency.source.commit) { throw new Error(
dependency.source = source; `Dependencies changed during convergence (${dependency.binding}); join writers and retry`,
changed = true; );
if (source && source.commit !== dependency.source.commit) {
dependency.source = source;
changed = true;
}
}
if (changed) {
const temporary = `${filename}.${randomUUID()}.tmp`;
try {
await writeFile(temporary, formatQuixosLockDocument(parsed.document), { flag: "wx" });
if ((await readFile(filename, "utf8")) !== before)
throw new Error(`Concurrent edit to ${file}; retry after joining writers`);
await rename(temporary, filename);
} finally {
await rm(temporary, { force: true });
}
}
} }
} const snapshotStarted = performance.now();
if (changed) { const commit = await snapshotCommit(root);
const temporary = `${filename}.${randomUUID()}.tmp`; if (process.env.QUIXOS_TRACE_CAPTURE === "1")
try { console.error(
await writeFile(temporary, formatQuixosLockDocument(parsed.document), { flag: "wx" }); `[${new Date().toISOString()}] Capture: ${directory} snapshot: ${Math.round(performance.now() - snapshotStarted)}ms`,
if ((await readFile(filename, "utf8")) !== before) );
throw new Error(`Concurrent edit to ${file}; retry after joining writers`); phase = "publication";
await rename(temporary, filename); const ref = retentionTagForCommit(commit);
} finally { const remote = await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref]);
await rm(temporary, { force: true }); if (remote && remote.split(/\s+/)[0] !== commit)
throw new Error(`Conflicting immutable retention ref ${ref}`);
if (!remote) {
await command(root, "git", ["push", node.source.repository, `${commit}:${ref}`]);
if (
(await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref])).split(/\s+/)[0] !==
commit
)
throw new Error("Published source retention was not observed");
} }
complete.set(directory, { ...node.source, commit });
} catch (error) {
blockers.push({ directory, phase, message: String(error).slice(0, 4000) });
} }
} return complete.has(directory);
const snapshotStarted = performance.now(); });
const commit = await snapshotCommit(root); })();
if (process.env.QUIXOS_TRACE_CAPTURE === "1") pending.set(directory, result);
console.error( return result;
`[${new Date().toISOString()}] Capture: ${directory} snapshot: ${Math.round(performance.now() - snapshotStarted)}ms`,
);
phase = "publication";
const ref = retentionTagForCommit(commit);
const remote = await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref]);
if (remote && remote.split(/\s+/)[0] !== commit) throw new Error(`Conflicting immutable retention ref ${ref}`);
if (!remote) await command(root, "git", ["push", node.source.repository, `${commit}:${ref}`]);
if ((await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref])).split(/\s+/)[0] !== commit)
throw new Error("Published source retention was not observed");
complete.set(directory, { ...node.source, commit });
} catch (error) {
blockers.push({ directory, phase, message: String(error).slice(0, 4000) });
}
active.delete(directory);
return complete.has(directory);
}; };
// Include newly created, not-yet-imported resources, then the root. // Include newly created, not-yet-imported resources, then the root.
if (target === "root") for (const directory of [...nodes.keys()].filter((d) => d !== "root")) await visit(directory); if (target === "root") await Promise.all([...nodes.keys()].filter((d) => d !== "root").map(visit));
await visit(target); await visit(target);
for (let index = blockers.length - 1; index >= 0; index--) for (let index = blockers.length - 1; index >= 0; index--)
if (!visited.has(blockers[index].directory)) blockers.splice(index, 1); if (!visited.has(blockers[index].directory)) blockers.splice(index, 1);
for (const [directory, source] of complete) { const recheckStarted = performance.now();
await mapConcurrent([...complete], async ([directory, source]) => {
try { try {
if ((await snapshotCommit(path.join(context.workbench, directory))) !== source.commit) if ((await snapshotCommit(path.join(context.workbench, directory))) !== source.commit)
throw new Error("Source advanced while converging; join writers and retry"); throw new Error("Source advanced while converging; join writers and retry");
} catch (error) { } catch (error) {
blockers.push({ directory, phase: "concurrent-edit", message: String(error) }); blockers.push({ directory, phase: "concurrent-edit", message: String(error) });
} }
} });
profile("capture-recheck", recheckStarted, { repositories: complete.size });
// Persist successful selections even if another repository is still broken. // Persist successful selections even if another repository is still broken.
// Recovery must not depend on all parents succeeding in the same invocation. // Recovery must not depend on all parents succeeding in the same invocation.
const graph = JSON.parse(graphBefore); const graph = JSON.parse(graphBefore);
@@ -238,11 +304,14 @@ export async function convergeAuthoring(start: string, target = "root") {
await rm(temporary, { force: true }); await rm(temporary, { force: true });
} }
} }
profile("capture", started, { repositories: nodes.size, snapshots: complete.size * 2, target });
return { return {
workbench: context.workbench, workbench: context.workbench,
converged: blockers.length === 0, converged: blockers.length === 0,
candidate: blockers.length ? null : (complete.get(target) ?? null), candidate: blockers.length ? null : (complete.get(target) ?? null),
retained: [...complete].map(([directory, source]) => ({ directory, source })), retained: [...complete]
.sort(([left], [right]) => left.localeCompare(right))
.map(([directory, source]) => ({ directory, source })),
worklist: blockers, worklist: blockers,
verificationEvidence: false, verificationEvidence: false,
activated: false, activated: false,
+72 -45
View File
@@ -13,13 +13,14 @@ export const checkerIdentity = () =>
export async function snapshotCommit(root: string): Promise<string> { export async function snapshotCommit(root: string): Promise<string> {
const run = async (...args: string[]) => const run = async (...args: string[]) =>
(await execFile("jj", args, { cwd: root, env: environment() })).stdout.trim(); (await execFile("jj", args, { cwd: root, env: environment() })).stdout.trim();
await run("status"); // A single normal log snapshots the working copy and observes both values
// jj resolve --list exits 1 on a clean revision. Query structured revision // from that exact revision. Separate status/log processes add no evidence.
// metadata instead of depending on diagnostic wording or swallowing errors. const [conflict, commit] = (await run("log", "--no-graph", "-r", "@", "-T", 'conflict ++ "\\n" ++ commit_id')).split(
if ((await run("--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "conflict")) !== "false") "\n",
throw new Error("Resolve source conflicts before verification"); );
const commit = await run("--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "commit_id"); if (conflict !== "false") throw new Error("Resolve source conflicts before verification");
if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit)) throw new Error("Verification requires an exact jj commit"); if (!commit || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit))
throw new Error("Verification requires an exact jj commit");
await execFile("git", ["diff", "--exit-code", "--no-ext-diff", "--no-textconv", commit, "--"], { await execFile("git", ["diff", "--exit-code", "--no-ext-diff", "--no-textconv", commit, "--"], {
cwd: root, cwd: root,
env: environment(), env: environment(),
@@ -110,6 +111,7 @@ export async function buildImmutableCandidate(
kind: "workspace" | "interface" | "package" | "build", kind: "workspace" | "interface" | "package" | "build",
logFile: string, logFile: string,
contractOnly = false, contractOnly = false,
observe?: (phase: "nixEvaluationMs" | "nixBuildMs", ms: number) => void,
): Promise<string> { ): Promise<string> {
if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(source.commit)) if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(source.commit))
throw new Error("An immutable candidate requires an exact commit"); throw new Error("An immutable candidate requires an exact commit");
@@ -126,16 +128,52 @@ export async function buildImmutableCandidate(
log.once("open", () => resolve()); log.once("open", () => resolve());
log.once("error", reject); log.once("error", reject);
}); });
return await new Promise((resolve, reject) => { const run = async (executable: string, args: string[]) =>
let output = "", await new Promise<string>((resolve, reject) => {
tail = "", let output = "",
failure: Error | undefined; tail = "",
const child = spawn( failure: Error | undefined;
"nix", const child = spawn(executable, args, { env: environment(), stdio: ["ignore", "pipe", "pipe"] });
[ const failedLog = (error: Error) => {
"build", failure = error;
"--impure", child.kill();
"--file", };
log.on("error", failedLog);
child.stdout.on("data", (chunk) => {
output += chunk;
});
child.stderr.on("data", (chunk) => {
log.write(chunk);
tail = (tail + String(chunk)).slice(-6000);
});
child.on("error", (error) => {
failure = error;
});
child.on("close", (code) => {
log.off("error", failedLog);
if (failure) reject(failure);
else if (code !== 0) reject(new Error(`Candidate Nix check failed (${code}). Full log: ${logFile}\n${tail}`));
else resolve(output.trim());
});
});
const timed = async (phase: "nixEvaluationMs" | "nixBuildMs", action: () => Promise<string>) => {
const started = performance.now();
try {
return await action();
} finally {
observe?.(phase, Math.round(performance.now() - started));
}
};
let gcDirectory: string | undefined;
try {
// Evaluation includes import-from-derivation builds (locks and contracts).
// Build the resulting derivation directly, without evaluating the DAG twice.
// Keep the derivation alive across the two processes, including concurrent
// GC. This temporary root is removed on success and on verification failure.
gcDirectory = await fs.mkdtemp(path.join(path.dirname(logFile), ".gc-root-"));
const gcRoot = path.join(gcDirectory, "candidate");
const derivation = await timed("nixEvaluationMs", async () => {
await run("nix-instantiate", [
builder, builder,
"--argstr", "--argstr",
"repository", "repository",
@@ -152,33 +190,22 @@ export async function buildImmutableCandidate(
"--arg", "--arg",
"contractOnly", "contractOnly",
contractOnly ? "true" : "false", contractOnly ? "true" : "false",
"--no-link", "--add-root",
"--print-out-paths", gcRoot,
"-L", "--indirect",
], ]);
{ env: environment(), stdio: ["ignore", "pipe", "pipe"] }, return await fs.readlink(gcRoot);
});
if (!/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+\.drv$/.test(derivation))
throw new Error("Nix returned an invalid candidate derivation");
const artifact = await timed("nixBuildMs", () =>
run("nix", ["build", derivation + "^*", "--no-link", "--print-out-paths", "-L"]),
); );
log.on("error", (error) => { if (!/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+$/.test(artifact))
failure = error; throw new Error("Nix returned an invalid candidate artifact");
child.kill(); return artifact;
}); } finally {
child.stdout.on("data", (chunk) => { await new Promise<void>((resolve) => log.end(resolve));
output += chunk; if (gcDirectory) await fs.rm(gcDirectory, { recursive: true, force: true });
}); }
child.stderr.on("data", (chunk) => {
log.write(chunk);
tail = (tail + String(chunk)).slice(-6000);
});
child.on("error", (error) => {
failure = error;
});
child.on("close", (code) =>
log.end(() => {
if (failure) reject(failure);
else if (code !== 0 || !/^\/nix\/store\/[a-z0-9]{32}-[^\s/]+$/.test(output.trim()))
reject(new Error(`Candidate Nix check failed (${code}). Full log: ${logFile}\n${tail}`));
else resolve(output.trim());
}),
);
});
} }
@@ -0,0 +1,40 @@
import { readFile } from "node:fs/promises";
import type { CheckedResourceResolver } from "./assembly.js";
import type { CapabilityResource } from "./parser.js";
/** Internal compiler inputs produced by checked-candidate.nix, not authoring
* receipts. The Nix composition owns their derivations and exact toolchain.
* Ordinary compiler callers omit this map and run all checks directly. */
export async function loadCheckedResources(filename?: string): Promise<CheckedResourceResolver | undefined> {
if (!filename) return undefined;
const entries = JSON.parse(await readFile(filename, "utf8")) as Array<{
kind: string;
repository: string;
commit: string;
artifact: string;
}>;
const resources = new Map<string, string>();
for (const entry of entries) {
const key = `${entry.kind}\0${entry.repository}\0${entry.commit}`;
if (resources.has(key)) throw new Error("Repeated checked resource input");
if (!/^\/nix\/store\/[^/]+$/.test(entry.artifact))
throw new Error("Checked resources must be immutable Nix outputs");
resources.set(key, entry.artifact);
}
return async (kind, source, lock, dependencies) => {
const artifact = resources.get(`${kind}\0${source.repository}\0${source.commit}`);
if (!artifact) return undefined;
const graph = JSON.parse(await readFile(`${artifact}/graph.json`, "utf8"));
const expected = [...dependencies].map(([binding, node]) => ({ binding, resourceKey: node.key }));
if (
graph.quixos.commit !== lock.quixos.commit ||
graph.quixos.repository !== lock.quixos.repository ||
graph.root.kind !== kind ||
graph.root.source.repository !== source.repository ||
graph.root.source.commit !== source.commit ||
JSON.stringify(graph.root.dependencies) !== JSON.stringify(expected)
)
throw new Error("Checked resource input does not match its locked dependency graph");
return JSON.parse(await readFile(`${artifact}/candidate.json`, "utf8")) as CapabilityResource;
};
}
@@ -80,6 +80,7 @@ export async function executionScaffold(
devShells.default = (import (helpers + "/quixos-package-helpers.nix")).mkCaminoYarnShell { inherit pkgs; }; devShells.default = (import (helpers + "/quixos-package-helpers.nix")).mkCaminoYarnShell { inherit pkgs; };
quixosPackages.checkedArtifacts = context: quixosPackages.checkedArtifacts = context:
let let
pkgs = camino.packageSet { inherit nixpkgs system context; };
contract = builtins.fromJSON (builtins.readFile context.buildContract); contract = builtins.fromJSON (builtins.readFile context.buildContract);
package = contract.definition.package; package = contract.definition.package;
portable = builtins.filter (entry: entry.execution.kind != "remote") package.exports; portable = builtins.filter (entry: entry.execution.kind != "remote") package.exports;
+5
View File
@@ -0,0 +1,5 @@
/** Observations belong in logs, never in content-addressed compiler outputs. */
export const profile = (phase: string, started: number, details: Record<string, unknown> = {}) => {
if (process.env.QUIXOS_TRACE_PROFILE === "1")
console.error(`[qx-profile] ${JSON.stringify({ phase, ms: Math.round(performance.now() - started), ...details })}`);
};
+5 -1
View File
@@ -6,11 +6,12 @@ import { bindingSchema } from "../bindings/index.js";
import { compileCapabilityResourceRepository, type ResolvedCapabilityResource } from "./assembly.js"; import { compileCapabilityResourceRepository, type ResolvedCapabilityResource } from "./assembly.js";
import { createGitCapabilityResolver } from "./git-resolver.js"; import { createGitCapabilityResolver } from "./git-resolver.js";
import { loadQueryGenerator } from "./query-generator.js"; import { loadQueryGenerator } from "./query-generator.js";
import { loadCheckedResources } from "./checked-resources.js";
const usage = `usage: quixos-resource-compile --root DIRECTORY --kind interface|package|build const usage = `usage: quixos-resource-compile --root DIRECTORY --kind interface|package|build
--repository URL --commit GIT_REV --checkout-root DIRECTORY --repository URL --commit GIT_REV --checkout-root DIRECTORY
[--snapshot-map PATH] [--snapshot-only true] [--graph-out PATH] [--schema-out PATH] [--snapshot-map PATH] [--snapshot-only true] [--graph-out PATH] [--schema-out PATH]
[--query-generator BUILD_MODULE_PATH] [--query-generator BUILD_MODULE_PATH] [--checked-resources NIX_MAP]
Resolves a standalone resource's recursive dependency-lock graph, validates Resolves a standalone resource's recursive dependency-lock graph, validates
its interface.qx or package.qx manifest, and emits the checked resource as JSON.`; its interface.qx or package.qx manifest, and emits the checked resource as JSON.`;
@@ -33,6 +34,7 @@ const parseArgs = (args: string[]) => {
"--graph-out", "--graph-out",
"--schema-out", "--schema-out",
"--query-generator", "--query-generator",
"--checked-resources",
].includes(key) || ].includes(key) ||
values.has(key) values.has(key)
) )
@@ -69,6 +71,7 @@ const parseArgs = (args: string[]) => {
graphOut: values.get("--graph-out"), graphOut: values.get("--graph-out"),
schemaOut: values.get("--schema-out"), schemaOut: values.get("--schema-out"),
queryGenerator: values.get("--query-generator"), queryGenerator: values.get("--query-generator"),
checkedResources: values.get("--checked-resources"),
} as const; } as const;
}; };
@@ -111,6 +114,7 @@ const main = async () => {
}, },
resolveResource, resolveResource,
generateQueries: await loadQueryGenerator(options.queryGenerator), generateQueries: await loadQueryGenerator(options.queryGenerator),
checkedResource: await loadCheckedResources(options.checkedResources),
}); });
if (options.graphOut) { if (options.graphOut) {
await writeFile( await writeFile(
+11 -1
View File
@@ -5,6 +5,8 @@ import process from "node:process";
import { compileWorkspaceRepository } from "./assembly.js"; import { compileWorkspaceRepository } from "./assembly.js";
import { createGitCapabilityResolver } from "./git-resolver.js"; import { createGitCapabilityResolver } from "./git-resolver.js";
import { loadQueryGenerator } from "./query-generator.js"; import { loadQueryGenerator } from "./query-generator.js";
import { loadCheckedResources } from "./checked-resources.js";
import { profile } from "./profile.js";
import { compileExecutionPlan, packageBuildContexts } from "../execution-world/index.js"; import { compileExecutionPlan, packageBuildContexts } from "../execution-world/index.js";
import { bindingSchema, specializeBindingSchema } from "../bindings/index.js"; import { bindingSchema, specializeBindingSchema } from "../bindings/index.js";
import { import {
@@ -18,7 +20,7 @@ const usage = `usage: quixos-workspace-compile --root DIRECTORY --checkout-root
[--snapshot-map PATH] [--graph-out PATH] [--workspace-id ID] [--workspace-revision-id ID] [--snapshot-map PATH] [--graph-out PATH] [--workspace-id ID] [--workspace-revision-id ID]
[--source-root-commit GIT_REV] [--baseline PLAN_JSON] [--evolution-out PATH] [--source-root-commit GIT_REV] [--baseline PLAN_JSON] [--evolution-out PATH]
[--reviews REVIEW_JSON] [--schemas-out PATH] [--execution-plan-out PATH] [--build-contexts-out PATH] [--reviews REVIEW_JSON] [--schemas-out PATH] [--execution-plan-out PATH] [--build-contexts-out PATH]
[--query-generator BUILD_MODULE_PATH] [--query-generator BUILD_MODULE_PATH] [--checked-resources NIX_MAP]
Resolves a workspace's recursive resource-lock graph, clones every exact Resolves a workspace's recursive resource-lock graph, clones every exact
resource revision, validates standalone interface/package manifests, and emits resource revision, validates standalone interface/package manifests, and emits
@@ -50,6 +52,7 @@ const parseArgs = (args: string[]) => {
evolutionOut: values.get("--evolution-out"), evolutionOut: values.get("--evolution-out"),
reviews: values.get("--reviews"), reviews: values.get("--reviews"),
queryGenerator: values.get("--query-generator"), queryGenerator: values.get("--query-generator"),
checkedResources: values.get("--checked-resources"),
}; };
}; };
@@ -63,6 +66,7 @@ const main = async () => {
checkoutRoot: options.checkoutRoot, checkoutRoot: options.checkoutRoot,
snapshotMap: options.snapshotMap, snapshotMap: options.snapshotMap,
}); });
const assemblyStarted = performance.now();
const assembled = await compileWorkspaceRepository({ const assembled = await compileWorkspaceRepository({
rootDirectory: options.rootDirectory, rootDirectory: options.rootDirectory,
workspaceId: options.workspaceId, workspaceId: options.workspaceId,
@@ -70,7 +74,9 @@ const main = async () => {
sourceRootCommit: options.sourceRootCommit, sourceRootCommit: options.sourceRootCommit,
resolveResource, resolveResource,
generateQueries: await loadQueryGenerator(options.queryGenerator), generateQueries: await loadQueryGenerator(options.queryGenerator),
checkedResource: await loadCheckedResources(options.checkedResources),
}); });
profile("workspace-assembly", assemblyStarted, { resources: assembled.resources.length });
if (options.graphOut) { if (options.graphOut) {
await writeFile( await writeFile(
@@ -110,12 +116,15 @@ const main = async () => {
} }
const candidate = { ...assembled.workspace, executionContracts: runtimeContracts(assembled.workspace) }; const candidate = { ...assembled.workspace, executionContracts: runtimeContracts(assembled.workspace) };
if (options.executionPlanOut || options.buildContextsOut) { if (options.executionPlanOut || options.buildContextsOut) {
const executionStarted = performance.now();
const plan = compileExecutionPlan(assembled.workspace); const plan = compileExecutionPlan(assembled.workspace);
if (options.executionPlanOut) await writeFile(options.executionPlanOut, `${JSON.stringify(plan, null, 2)}\n`); if (options.executionPlanOut) await writeFile(options.executionPlanOut, `${JSON.stringify(plan, null, 2)}\n`);
if (options.buildContextsOut) if (options.buildContextsOut)
await writeFile(options.buildContextsOut, JSON.stringify(packageBuildContexts(plan.definition.bindingSchema))); await writeFile(options.buildContextsOut, JSON.stringify(packageBuildContexts(plan.definition.bindingSchema)));
profile("execution-plan-and-contexts", executionStarted);
} }
if (options.schemasOut) { if (options.schemasOut) {
const schemasStarted = performance.now();
const schemas: Record<string, unknown> = {}; const schemas: Record<string, unknown> = {};
for (const node of assembled.capabilityResources.filter((entry) => entry.kind === "package")) { for (const node of assembled.capabilityResources.filter((entry) => entry.kind === "package")) {
const closure = new Map<string, typeof node>(); const closure = new Map<string, typeof node>();
@@ -133,6 +142,7 @@ const main = async () => {
); );
} }
await writeFile(options.schemasOut, JSON.stringify(schemas)); await writeFile(options.schemasOut, JSON.stringify(schemas));
profile("package-schemas", schemasStarted);
} }
if (options.evolutionOut) { if (options.evolutionOut) {
const baseline = options.baseline const baseline = options.baseline
+63 -5
View File
@@ -107,6 +107,35 @@ package Runtime id "package:runtime" revision "package:runtime@1" {
assert.equal(resource.resource.kind, "package"); assert.equal(resource.resource.kind, "package");
assert.equal(resource.resources.length, 2); assert.equal(resource.resources.length, 2);
assert.equal(resource.directResources.size, 1); assert.equal(resource.directResources.size, 1);
const checked = new Map(result.resources.map((node) => [node.key, node.resource]));
let generated = 0;
const reused = await compileWorkspaceRepository({
rootDirectory: root,
resolveResource: async (source, kind) => ({
directory: directories.get(`${kind}\0${source.repository}\0${source.commit}`)!,
}),
checkedResource: async (kind, source) => {
const value = checked.get(`${kind}\0${source.repository}\0${source.commit}`);
return value?.kind === "build" ? undefined : value;
},
generateQueries: async () => {
generated++;
return { queries: [], documents: new Map() };
},
});
assert.deepEqual(reused, result, "Nix-composed resource checks preserve full assembly results");
assert.equal(generated, 0, "unchanged checked packages do not run their query frontend again");
await assert.rejects(
compileWorkspaceRepository({
rootDirectory: root,
resolveResource: async (source, kind) => ({
directory: directories.get(`${kind}\0${source.repository}\0${source.commit}`)!,
}),
checkedResource: async () => (resource.resource.kind === "build" ? undefined : resource.resource),
}),
/Checked resource identity mismatch/,
);
}); });
test("resource repositories cannot own workspace Quixos selection policy", async (context) => { test("resource repositories cannot own workspace Quixos selection policy", async (context) => {
@@ -249,19 +278,48 @@ test("resource-owned fragment imports compile through the locked package closure
}`, }`,
); );
await writeFile(path.join(dirs.screen!, "screen.graphql"), "query Screen {root {...Row}}"); await writeFile(path.join(dirs.screen!, "screen.graphql"), "query Screen {root {...Row}}");
const result = await compileCapabilityResourceRepository({ const options = {
rootDirectory: dirs.screen!, rootDirectory: dirs.screen!,
kind: "package", kind: "package" as const,
source: { resolver: "git", repository: "https://example.test/screen.git", commit: packageCommit }, source: { resolver: "git" as const, repository: "https://example.test/screen.git", commit: packageCommit },
resolveResource: async (source) => { resolveResource: async (source: { repository: string }) => {
const name = new URL(source.repository).pathname.slice(1, -4); const name = new URL(source.repository).pathname.slice(1, -4);
assert.ok(dirs[name], name); assert.ok(dirs[name], name);
return { directory: dirs[name]! }; return { directory: dirs[name]! };
}, },
}); };
const result = await compileCapabilityResourceRepository(options);
assert.equal(result.resource.kind, "package"); assert.equal(result.resource.kind, "package");
if (result.resource.kind !== "package") throw Error("expected package"); if (result.resource.kind !== "package") throw Error("expected package");
const view = result.resource.revision.checkedQueries!.find((q) => q.declaration.id === "view")!; const view = result.resource.revision.checkedQueries!.find((q) => q.declaration.id === "view")!;
assert.equal(view.selection[0]!.selection[0]!.fragmentSpread?.package, "rows@1"); assert.equal(view.selection[0]!.selection[0]!.fragmentSpread?.package, "rows@1");
assert.equal(result.resources.length, 3); assert.equal(result.resources.length, 3);
const checked = new Map(result.resources.map((node) => [node.key, node.resource]));
let generated = 0;
const reused = await compileCapabilityResourceRepository({
...options,
checkedResource: async (kind, source) => {
if (source.repository === options.source.repository) return undefined;
const resource = checked.get(`${kind}\0${source.repository}\0${source.commit}`);
return resource?.kind === "build" ? undefined : resource;
},
generateQueries: async () => {
generated++;
return { queries: [], documents: new Map() };
},
});
assert.deepEqual(reused, result, "cached imported fragments and full checking produce identical contracts");
assert.equal(generated, 1, "only the uncached package checks its queries");
await writeFile(path.join(dirs.screen!, "screen.graphql"), "query Screen {root {unknownField}}");
await assert.rejects(
compileCapabilityResourceRepository({
...options,
source: { ...options.source, commit: "5".repeat(40) },
checkedResource: async (kind, source) => {
const resource = checked.get(`${kind}\0${source.repository}\0${source.commit}`);
return resource?.kind === "build" ? undefined : resource;
},
}),
/unknownField/,
);
}); });