import fs from "node:fs/promises"; import path from "node:path"; import os from "node:os"; import { randomUUID } from "node:crypto"; import { execFile as callback } from "node:child_process"; import { promisify } from "node:util"; import { loadQuixosLock, parseQuixosLockDocument } from "../resource-lock/index.js"; import { contentDigest } from "../capability-model/evolution.js"; import { planStructure, applyStructure, type StructuralRequest } from "./structural-plan.js"; import { snapshotRepository } from "./candidate-check.js"; import { compileWorkspaceRepository, compileCapabilityResourceRepository, type ResolvedCapabilityResource, } from "./assembly.js"; import { createGitCapabilityResolver } from "./git-resolver.js"; import { snapshotCommit, buildImmutableCandidate } from "./checked-build.js"; import { planEvolution, reviewSemanticMajors, type WorkspaceRevision, type EvolutionReview, } from "../capability-model/index.js"; import { withFileLock } from "./file-lock.js"; const execFile = promisify(callback); type Source = { repository: string; commit: string }; export type UpgradeNode = { kind: "workspace" | "package" | "interface" | "build"; directory: string; source: Source }; export type UpgradeSpec = { nodes: UpgradeNode[]; quixos?: Source; baseline?: string; reviews?: string; bootstrap?: boolean; }; type NodePlan = UpgradeNode & { treeDigest: string; dependencies: string[]; lockFiles: string[] }; export type UpgradePlan = { schemaVersion: 1; workbench: string; spec: UpgradeSpec; nodes: NodePlan[]; digest: string }; type Step = { directory: string; phase: "editing" | "prepared" | "refactor" | "checked" | "publishing" | "published"; commit?: string; treeDigest?: string; structuralJournal?: string; structuralPlan?: Awaited>; }; type Journal = { schemaVersion: 1; plan: UpgradePlan; steps: Step[] }; const sourceKey = (node: { kind: string; source: Source }) => JSON.stringify([node.kind, node.source.repository, node.source.commit]); const command = async (cwd: string, tool: string, args: string[]) => ( await execFile(tool, args, { cwd, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, GIT_TERMINAL_PROMPT: "0", QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_SUBTREE_PUBLISH: "0" }, }) ).stdout.trim(); const validSource = (value: Source) => { const url = new URL(value.repository); if ( url.protocol !== "https:" || url.username || url.password || url.search || url.hash || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(value.commit) ) throw new Error("Upgrade sources must be exact credential-free HTTPS revisions"); }; const location = async (root: string, directory: string) => { if (directory !== "root" && !/^resources\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(directory)) throw new Error("Upgrade target must be a managed root/resource repository"); const resolved = await fs.realpath(path.join(root, directory)); if (resolved !== path.join(root, directory)) throw new Error("Upgrade target crosses a symlink"); return resolved; }; const treeDigest = async (root: string) => { const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "qx-upgrade-tree-")); try { return (await snapshotRepository(root, temporary)).treeDigest; } finally { await fs.rm(temporary, { recursive: true, force: true }); } }; const writeJournal = async (filename: string, journal: unknown) => { const temp = `${filename}.${randomUUID()}.tmp`; const handle = await fs.open(temp, "wx", 0o600); try { await handle.writeFile(JSON.stringify(journal, null, 2)); await handle.sync(); } finally { await handle.close(); } await fs.rename(temp, filename); const directory = await fs.open(path.dirname(filename), "r"); try { await directory.sync(); } finally { await directory.close(); } }; export const discoverUpgradeSpec = async (workbench: string): Promise => { const graph = JSON.parse(await fs.readFile(path.join(workbench, ".quixos/resource-graph.json"), "utf8")); const root = await location(workbench, "root"); const nodes: UpgradeNode[] = [ { kind: "workspace", directory: "root", source: { repository: await command(root, "git", ["config", "--get", "remote.origin.url"]), commit: await command(root, "jj", ["--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "commit_id"]), }, }, ...graph.resources.map((entry: { kind: "interface" | "package" | "build"; directory: string; source: Source }) => ({ kind: entry.kind, source: entry.source, directory: path.relative(workbench, path.resolve(workbench, entry.directory)), })), ]; let baseline: string | undefined; try { const host = JSON.parse(await fs.readFile("/etc/quixos/workspace-source.json", "utf8")); if ((await fs.realpath(host.workbenchRoot)) === (await fs.realpath(workbench))) baseline = JSON.parse( await fs.readFile(path.join(host.runtimeClosureRoot, "manifest.json"), "utf8"), ).workspacePlanPath; } catch (error) { if (!["ENOENT", "EACCES"].includes((error as NodeJS.ErrnoException).code ?? "")) throw error; } return { nodes, baseline }; }; /** Read-only source plan. Repositories are selected explicitly, including any * parallel versions of the same resource; no guesses at a floating 'latest'. */ export const planPinUpgrades = async (workbenchPath: string, spec: UpgradeSpec): Promise => { const workbench = await fs.realpath(workbenchPath); if ( !Array.isArray(spec.nodes) || !spec.nodes.length || spec.nodes.length > 100 || spec.nodes.filter((node) => node.kind === "workspace").length !== 1 ) throw new Error("Upgrade graph requires one workspace and at most 100 repositories"); if (spec.quixos) validSource(spec.quixos); const keys = new Map(); for (const node of spec.nodes) { validSource(node.source); if (!["workspace", "package", "interface", "build"].includes(node.kind) || keys.has(sourceKey(node))) throw new Error("Duplicate/invalid upgrade resource identity"); keys.set(sourceKey(node), node.directory); } if (new Set(spec.nodes.map((node) => node.directory)).size !== spec.nodes.length) throw new Error("Upgrade directories must be distinct"); const nodes: NodePlan[] = []; for (const node of spec.nodes) { const root = await location(workbench, node.directory); if ((await command(root, "git", ["config", "--get", "remote.origin.url"])) !== node.source.repository) throw new Error(`Upgrade origin differs from selected source: ${node.directory}`); const loaded = await loadQuixosLock(path.join(root, "quixos.lock")); if (!loaded.ok) throw new Error( `Invalid lock in ${node.directory}: ${loaded.diagnostics.map((entry) => entry.message).join("; ")}`, ); const dependencies = loaded.lock.resources .map((resource) => keys.get(sourceKey(resource))) .filter((value): value is string => Boolean(value)); nodes.push({ ...node, treeDigest: await treeDigest(root), dependencies: [...new Set(dependencies)], lockFiles: loaded.lock.sourceFiles ?? ["quixos.lock"], }); } const ordered: NodePlan[] = [], remaining = [...nodes]; while (remaining.length) { const index = remaining.findIndex((node) => node.dependencies.every((dependency) => ordered.some((entry) => entry.directory === dependency)), ); if (index < 0) throw new Error("Cyclic source publication graph"); ordered.push(remaining.splice(index, 1)[0]); } const workspace = ordered.find((node) => node.kind === "workspace")!; // Even unreferenced new resources are published before the root. ordered.splice(ordered.indexOf(workspace), 1); ordered.push(workspace); const plan = { schemaVersion: 1 as const, workbench, spec, nodes: ordered }; return { ...plan, digest: contentDigest(plan) }; }; export type UpgradeEffects = { check(node: NodePlan, root: string, output: string, spec: UpgradeSpec): Promise; snapshot(root: string): Promise; publish(root: string, commit: string): Promise; }; /** Review a successfully built immutable candidate using the same boundary as * ordinary authoring checks. Execution storage compatibility is checked against * the current graph at activation; the legacy evolution model is not its schema. */ export async function reviewUpgradeCandidate({ artifact, baseline, candidate, reviews = [], }: { artifact: string; baseline: WorkspaceRevision | null; candidate: WorkspaceRevision; reviews?: EvolutionReview[]; }) { let execution = false; try { await fs.access(path.join(artifact, "execution-plan.json")); execution = true; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } return execution ? { filename: "semantic-review.json", review: reviewSemanticMajors(baseline, candidate, reviews) } : { filename: "evolution.json", review: planEvolution(baseline, candidate, { reviews }) }; } const effects: UpgradeEffects = { async check(node, root, output, spec) { if (node.kind === "workspace" && !spec.baseline && !spec.bootstrap) throw new Error( "Upgrading a workspace requires its checked active baseline for major-review checks (or explicit bootstrap:true for a new workspace)", ); // Explicit baseline upgrades use the same immutable Nix checker. Retaining // an unverified source is safe and must precede a remote flake fetch. await fs.mkdir(output); const commit = await snapshotCommit(root); await effects.publish(root, commit); const artifact = await buildImmutableCandidate({ ...node.source, commit }, node.kind, path.join(output, "nix.log")); const candidate = await fs.readFile(path.join(artifact, "candidate.json"), "utf8"); await fs.writeFile(path.join(output, "candidate.json"), candidate); if (node.kind === "workspace") { const baseline = spec.baseline ? (JSON.parse(await fs.readFile(spec.baseline, "utf8")) as WorkspaceRevision) : null; const reviews = spec.reviews ? (JSON.parse(await fs.readFile(spec.reviews, "utf8")) as EvolutionReview[]) : []; const { filename, review } = await reviewUpgradeCandidate({ artifact, baseline, candidate: JSON.parse(candidate), reviews, }); await fs.writeFile(path.join(output, filename), JSON.stringify(review, null, 2)); if (review.blockers.length) throw new Error(`Refactor required in ${node.directory}: ${review.blockers.join("; ")}`); } }, async snapshot(root) { const commit = await snapshotCommit(root); if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit)) throw new Error("Publication did not resolve an exact commit"); await command(root, "git", ["diff", "--exit-code", "--no-ext-diff", "--no-textconv", commit, "--"]); const tracked = new Set((await command(root, "git", ["ls-tree", "-r", "--name-only", "-z", commit])).split("\0")); for (const file of (await command(root, "git", ["ls-files", "--others", "--exclude-standard", "-z"])) .split("\0") .filter(Boolean)) if (!tracked.has(file)) throw new Error(`Uncaptured source file ${file}`); return commit; }, async publish(root, commit) { const ref = `refs/tags/quixos-reachability/${commit}`; const remote = await command(root, "git", ["ls-remote", "--refs", "origin", ref]); if (remote && remote !== `${commit}\t${ref}`) throw new Error("Immutable publication ref conflict"); if (!remote) await command(root, "git", ["push", "origin", `${commit}:${ref}`]); if ((await command(root, "git", ["ls-remote", "--refs", "origin", ref])) !== `${commit}\t${ref}`) throw new Error("Publication response uncertain; retry the same journal"); }, }; /** Explicit --publish only. Append-only remote retention; never moves the * workspace branch, activates code, or rolls back previously published nodes. */ export const applyPinUpgrades = async ( plan: UpgradePlan, journalId?: string, implementation: UpgradeEffects = effects, options: { acceptEdits?: boolean } = {}, ) => { if (implementation === effects && !plan.spec.baseline && !plan.spec.bootstrap) throw new Error("Publication requires an active checked baseline or explicit bootstrap:true"); const { digest, ...body } = plan; if (contentDigest(body) !== digest) throw new Error("Upgrade plan digest mismatch"); const directory = path.join(plan.workbench, ".quixos", "upgrades"); await fs.mkdir(directory, { recursive: true, mode: 0o700 }); if ((await fs.realpath(directory)) !== directory) throw new Error("Upgrade journals must not cross symlinks"); const id = journalId ?? randomUUID(); if (!/^[a-f0-9-]{36}$/.test(id)) throw new Error("Invalid upgrade journal ID"); const filename = path.join(directory, `${id}.json`); return withFileLock(path.join(directory, "writer.lock"), async () => { try { const journal: Journal = journalId ? JSON.parse(await fs.readFile(filename, "utf8")) : { schemaVersion: 1, plan, steps: [] }; if (journal.plan.digest !== plan.digest) throw new Error("Upgrade journal belongs to another plan"); if (!journalId) await writeJournal(filename, journal); for (const node of plan.nodes) { const root = await location(plan.workbench, node.directory); if ((await command(root, "git", ["config", "--get", "remote.origin.url"])) !== node.source.repository) throw new Error("Upgrade remote changed after planning"); let step = journal.steps.find((entry) => entry.directory === node.directory); if (step?.phase === "published") continue; if (!step) { if ((await treeDigest(root)) !== node.treeDigest) throw new Error(`Stale upgrade plan: ${node.directory}`); const files: StructuralRequest["files"] = []; for (const file of node.lockFiles) { const parsed = parseQuixosLockDocument(await fs.readFile(path.join(root, file), "utf8")); if (!parsed.ok) throw new Error("Invalid lock during upgrade"); const edits = []; for (const resource of parsed.document.resources) { const dependency = plan.nodes.find((entry) => sourceKey(entry) === sourceKey(resource)); const published = dependency && journal.steps.find((entry) => entry.directory === dependency.directory && entry.phase === "published"); if (published?.commit && published.commit !== resource.source.commit) edits.push({ operation: "dependency" as const, kind: resource.kind, name: resource.binding, source: { ...resource.source, commit: published.commit }, }); } if (parsed.document.kind === "root" && plan.spec.quixos) edits.push({ operation: "quixos-pin" as const, source: plan.spec.quixos }); if (edits.length) files.push({ file, edits }); } const structuralPlan = files.length ? await planStructure( root, { kind: node.kind, source: node.source, files }, process.env.QUIXOS_SNAPSHOT_MAP, ) : undefined; step = { directory: node.directory, phase: "editing", treeDigest: node.treeDigest, structuralPlan, structuralJournal: structuralPlan ? randomUUID() : undefined, }; journal.steps.push(step); await writeJournal(filename, journal); } if (step.phase === "editing") { if (step.structuralPlan) await applyStructure(step.structuralPlan, step.structuralJournal); else if ((await treeDigest(root)) !== step.treeDigest) throw new Error("Source changed before upgrade editing"); step.treeDigest = await treeDigest(root); step.phase = "prepared"; await writeJournal(filename, journal); } if (step.phase === "refactor") { const current = await treeDigest(root); if (current !== step.treeDigest && !options.acceptEdits) throw new Error("Refactored source requires --accept-edits when resuming"); step.treeDigest = current; step.phase = "prepared"; await writeJournal(filename, journal); } if ((await treeDigest(root)) !== step.treeDigest) throw new Error(`Source changed during upgrade: ${node.directory}; inspect ${filename}`); if (step.phase === "prepared") { try { await implementation.check( node, root, path.join(directory, `${id}-${node.directory.replaceAll("/", "-")}-${randomUUID()}`), plan.spec, ); } catch (error) { step.phase = "refactor"; await writeJournal(filename, journal); throw error; } if ((await treeDigest(root)) !== step.treeDigest) throw new Error("Source changed while checking"); step.phase = "checked"; await writeJournal(filename, journal); } if (step.phase === "checked") { step.commit = await implementation.snapshot(root); if ((await treeDigest(root)) !== step.treeDigest) throw new Error("Publication snapshot changed checked files"); step.phase = "publishing"; await writeJournal(filename, journal); } await implementation.publish(root, step.commit!); step.phase = "published"; await writeJournal(filename, journal); } // Keep subsequent automatic upgrades associated with the newly published // identities, without renaming repositories or changing any selected branch. // Explicit-spec callers without a managed graph retain the journal as their // source of revisions instead. const graphFile = path.join(plan.workbench, ".quixos/resource-graph.json"); let graphText: string | undefined; try { graphText = await fs.readFile(graphFile, "utf8"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } if (graphText !== undefined) { const previous = JSON.parse(graphText); const snapshots = await Promise.all( previous.resources.map(async (entry: { kind: string; source: Source; directory: string }) => ({ kind: entry.kind, ...entry.source, directory: await location( plan.workbench, path.relative(plan.workbench, path.resolve(plan.workbench, entry.directory)), ), })), ); for (const node of plan.nodes.filter((entry) => entry.kind !== "workspace")) { const step = journal.steps.find((entry) => entry.directory === node.directory)!; if ((await treeDigest(await location(plan.workbench, node.directory))) !== step.treeDigest) throw new Error("Published source changed before workbench graph refresh"); snapshots.push({ kind: node.kind, repository: node.source.repository, commit: step.commit, directory: path.join(plan.workbench, node.directory), }); } const unique = [ ...new Map( snapshots.map((entry: { kind: string; repository: string; commit: string }) => [ JSON.stringify([entry.kind, entry.repository, entry.commit]), entry, ]), ).values(), ]; const snapshotMap = path.join(directory, `${id}-published-snapshots.json`); await fs.writeFile(snapshotMap, JSON.stringify({ resources: unique })); const resolveResource = await createGitCapabilityResolver({ checkoutRoot: path.join(directory, `${id}-graph-resources`), snapshotMap, snapshotOnly: true, }); const rootNode = plan.nodes.find((node) => node.kind === "workspace")!; if ( (await treeDigest(await location(plan.workbench, rootNode.directory))) !== journal.steps.find((step) => step.directory === rootNode.directory)!.treeDigest ) throw new Error("Root source changed before workbench graph refresh"); const compiled = await compileWorkspaceRepository({ rootDirectory: await location(plan.workbench, rootNode.directory), resolveResource, }); // Managed repositories need not currently be reachable from the workspace. // Keep them discoverable/checkpointed until explicitly removed by the user. const resources = new Map( compiled.resources.map((node) => [node.key, node]), ); for (const node of plan.nodes.filter((entry) => entry.kind !== "workspace")) { const step = journal.steps.find((entry) => entry.directory === node.directory)!; const source = { resolver: "git" as const, repository: node.source.repository, commit: step.commit! }; const key = `${node.kind}\0${source.repository}\0${source.commit}`; if (resources.has(key)) continue; const directory = await location(plan.workbench, node.directory); const standalone = await compileCapabilityResourceRepository({ rootDirectory: directory, kind: node.kind as "package" | "interface" | "build", source, resolveResource, }); for (const dependency of standalone.resources) resources.set(dependency.key, dependency); resources.set(key, { key, kind: node.kind as "package" | "interface" | "build", source, directory, lock: standalone.lock, resource: standalone.resource, dependencies: standalone.directResources, }); } await writeJournal(graphFile, { formatVersion: 1, quixos: compiled.lock.quixos, directResources: [...compiled.directResources.entries()].map(([bindingKey, node]) => { const [kind, binding] = bindingKey.split("\0"); return { kind, binding, resourceKey: node.key, directory: path.relative(plan.workbench, node.directory) }; }), resources: [...resources.values()].map((node) => ({ key: node.key, kind: node.kind, source: node.source, directory: path.relative(plan.workbench, node.directory), ...(node.resource.kind === "build" ? {} : { resourceId: node.resource.kind === "interface" ? node.resource.revision.interfaceId : node.resource.revision.packageId, revisionId: node.resource.revision.revisionId, }), dependencies: [...node.dependencies.entries()].map(([binding, dependency]) => ({ binding, resourceKey: dependency.key, })), })), }); } return { id, journal: filename, revisions: journal.steps.map((step) => ({ directory: step.directory, commit: step.commit })), activated: false, }; } catch (error) { throw new Error(`${error instanceof Error ? error.message : String(error)}; upgrade journal ${filename}`, { cause: error, }); } }); };