quixos-protocol/src/capability-language/pin-upgrades.ts

import fs from "node:fs/promises";
import path from "node:path";
import os from "node:os";
import { randomUUID } from "node:crypto";
import { execFile as callback } from "node:child_process";
import { promisify } from "node:util";
import { loadQuixosLock, parseQuixosLockDocument } from "../resource-lock/index.js";
import { contentDigest } from "../capability-model/evolution.js";
import { planStructure, applyStructure, type StructuralRequest } from "./structural-plan.js";
import { snapshotRepository } from "./candidate-check.js";
import {
  compileWorkspaceRepository,
  compileCapabilityResourceRepository,
  type ResolvedCapabilityResource,
} from "./assembly.js";
import { createGitCapabilityResolver } from "./git-resolver.js";
import { snapshotCommit, buildImmutableCandidate } from "./checked-build.js";
import {
  planEvolution,
  reviewSemanticMajors,
  type WorkspaceRevision,
  type EvolutionReview,
} from "../capability-model/index.js";
import { withFileLock } from "./file-lock.js";
const execFile = promisify(callback);
type Source = { repository: string; commit: string };
export type UpgradeNode = { kind: "workspace" | "package" | "interface" | "build"; directory: string; source: Source };
export type UpgradeSpec = {
  nodes: UpgradeNode[];
  quixos?: Source;
  baseline?: string;
  reviews?: string;
  bootstrap?: boolean;
};
type NodePlan = UpgradeNode & { treeDigest: string; dependencies: string[]; lockFiles: string[] };
export type UpgradePlan = { schemaVersion: 1; workbench: string; spec: UpgradeSpec; nodes: NodePlan[]; digest: string };
type Step = {
  directory: string;
  phase: "editing" | "prepared" | "refactor" | "checked" | "publishing" | "published";
  commit?: string;
  treeDigest?: string;
  structuralJournal?: string;
  structuralPlan?: Awaited<ReturnType<typeof planStructure>>;
};
type Journal = { schemaVersion: 1; plan: UpgradePlan; steps: Step[] };
const sourceKey = (node: { kind: string; source: Source }) =>
  JSON.stringify([node.kind, node.source.repository, node.source.commit]);
const command = async (cwd: string, tool: string, args: string[]) =>
  (
    await execFile(tool, args, {
      cwd,
      maxBuffer: 16 * 1024 * 1024,
      env: { ...process.env, GIT_TERMINAL_PROMPT: "0", QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_SUBTREE_PUBLISH: "0" },
    })
  ).stdout.trim();
const validSource = (value: Source) => {
  const url = new URL(value.repository);
  if (
    url.protocol !== "https:" ||
    url.username ||
    url.password ||
    url.search ||
    url.hash ||
    !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(value.commit)
  )
    throw new Error("Upgrade sources must be exact credential-free HTTPS revisions");
};
const location = async (root: string, directory: string) => {
  if (directory !== "root" && !/^resources\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(directory))
    throw new Error("Upgrade target must be a managed root/resource repository");
  const resolved = await fs.realpath(path.join(root, directory));
  if (resolved !== path.join(root, directory)) throw new Error("Upgrade target crosses a symlink");
  return resolved;
};
const treeDigest = async (root: string) => {
  const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "qx-upgrade-tree-"));
  try {
    return (await snapshotRepository(root, temporary)).treeDigest;
  } finally {
    await fs.rm(temporary, { recursive: true, force: true });
  }
};
const writeJournal = async (filename: string, journal: unknown) => {
  const temp = `${filename}.${randomUUID()}.tmp`;
  const handle = await fs.open(temp, "wx", 0o600);
  try {
    await handle.writeFile(JSON.stringify(journal, null, 2));
    await handle.sync();
  } finally {
    await handle.close();
  }
  await fs.rename(temp, filename);
  const directory = await fs.open(path.dirname(filename), "r");
  try {
    await directory.sync();
  } finally {
    await directory.close();
  }
};

export const discoverUpgradeSpec = async (workbench: string): Promise<UpgradeSpec> => {
  const graph = JSON.parse(await fs.readFile(path.join(workbench, ".quixos/resource-graph.json"), "utf8"));
  const root = await location(workbench, "root");
  const nodes: UpgradeNode[] = [
    {
      kind: "workspace",
      directory: "root",
      source: {
        repository: await command(root, "git", ["config", "--get", "remote.origin.url"]),
        commit: await command(root, "jj", ["--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "commit_id"]),
      },
    },
    ...graph.resources.map((entry: { kind: "interface" | "package" | "build"; directory: string; source: Source }) => ({
      kind: entry.kind,
      source: entry.source,
      directory: path.relative(workbench, path.resolve(workbench, entry.directory)),
    })),
  ];
  let baseline: string | undefined;
  try {
    const host = JSON.parse(await fs.readFile("/etc/quixos/workspace-source.json", "utf8"));
    if ((await fs.realpath(host.workbenchRoot)) === (await fs.realpath(workbench)))
      baseline = JSON.parse(
        await fs.readFile(path.join(host.runtimeClosureRoot, "manifest.json"), "utf8"),
      ).workspacePlanPath;
  } catch (error) {
    if (!["ENOENT", "EACCES"].includes((error as NodeJS.ErrnoException).code ?? "")) throw error;
  }
  return { nodes, baseline };
};

/** Read-only source plan. Repositories are selected explicitly, including any
 * parallel versions of the same resource; no guesses at a floating 'latest'. */
export const planPinUpgrades = async (workbenchPath: string, spec: UpgradeSpec): Promise<UpgradePlan> => {
  const workbench = await fs.realpath(workbenchPath);
  if (
    !Array.isArray(spec.nodes) ||
    !spec.nodes.length ||
    spec.nodes.length > 100 ||
    spec.nodes.filter((node) => node.kind === "workspace").length !== 1
  )
    throw new Error("Upgrade graph requires one workspace and at most 100 repositories");
  if (spec.quixos) validSource(spec.quixos);
  const keys = new Map<string, string>();
  for (const node of spec.nodes) {
    validSource(node.source);
    if (!["workspace", "package", "interface", "build"].includes(node.kind) || keys.has(sourceKey(node)))
      throw new Error("Duplicate/invalid upgrade resource identity");
    keys.set(sourceKey(node), node.directory);
  }
  if (new Set(spec.nodes.map((node) => node.directory)).size !== spec.nodes.length)
    throw new Error("Upgrade directories must be distinct");
  const nodes: NodePlan[] = [];
  for (const node of spec.nodes) {
    const root = await location(workbench, node.directory);
    if ((await command(root, "git", ["config", "--get", "remote.origin.url"])) !== node.source.repository)
      throw new Error(`Upgrade origin differs from selected source: ${node.directory}`);
    const loaded = await loadQuixosLock(path.join(root, "quixos.lock"));
    if (!loaded.ok)
      throw new Error(
        `Invalid lock in ${node.directory}: ${loaded.diagnostics.map((entry) => entry.message).join("; ")}`,
      );
    const dependencies = loaded.lock.resources
      .map((resource) => keys.get(sourceKey(resource)))
      .filter((value): value is string => Boolean(value));
    nodes.push({
      ...node,
      treeDigest: await treeDigest(root),
      dependencies: [...new Set(dependencies)],
      lockFiles: loaded.lock.sourceFiles ?? ["quixos.lock"],
    });
  }
  const ordered: NodePlan[] = [],
    remaining = [...nodes];
  while (remaining.length) {
    const index = remaining.findIndex((node) =>
      node.dependencies.every((dependency) => ordered.some((entry) => entry.directory === dependency)),
    );
    if (index < 0) throw new Error("Cyclic source publication graph");
    ordered.push(remaining.splice(index, 1)[0]);
  }
  const workspace = ordered.find((node) => node.kind === "workspace")!;
  // Even unreferenced new resources are published before the root.
  ordered.splice(ordered.indexOf(workspace), 1);
  ordered.push(workspace);
  const plan = { schemaVersion: 1 as const, workbench, spec, nodes: ordered };
  return { ...plan, digest: contentDigest(plan) };
};

export type UpgradeEffects = {
  check(node: NodePlan, root: string, output: string, spec: UpgradeSpec): Promise<void>;
  snapshot(root: string): Promise<string>;
  publish(root: string, commit: string): Promise<void>;
};

/** Review a successfully built immutable candidate using the same boundary as
 * ordinary authoring checks. Execution storage compatibility is checked against
 * the current graph at activation; the legacy evolution model is not its schema.
 */
export async function reviewUpgradeCandidate({
  artifact,
  baseline,
  candidate,
  reviews = [],
}: {
  artifact: string;
  baseline: WorkspaceRevision | null;
  candidate: WorkspaceRevision;
  reviews?: EvolutionReview[];
}) {
  let execution = false;
  try {
    await fs.access(path.join(artifact, "execution-plan.json"));
    execution = true;
  } catch (error) {
    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
  }
  return execution
    ? { filename: "semantic-review.json", review: reviewSemanticMajors(baseline, candidate, reviews) }
    : { filename: "evolution.json", review: planEvolution(baseline, candidate, { reviews }) };
}

const effects: UpgradeEffects = {
  async check(node, root, output, spec) {
    if (node.kind === "workspace" && !spec.baseline && !spec.bootstrap)
      throw new Error(
        "Upgrading a workspace requires its checked active baseline for major-review checks (or explicit bootstrap:true for a new workspace)",
      );
    // Explicit baseline upgrades use the same immutable Nix checker. Retaining
    // an unverified source is safe and must precede a remote flake fetch.
    await fs.mkdir(output);
    const commit = await snapshotCommit(root);
    await effects.publish(root, commit);
    const artifact = await buildImmutableCandidate({ ...node.source, commit }, node.kind, path.join(output, "nix.log"));
    const candidate = await fs.readFile(path.join(artifact, "candidate.json"), "utf8");
    await fs.writeFile(path.join(output, "candidate.json"), candidate);
    if (node.kind === "workspace") {
      const baseline = spec.baseline
        ? (JSON.parse(await fs.readFile(spec.baseline, "utf8")) as WorkspaceRevision)
        : null;
      const reviews = spec.reviews ? (JSON.parse(await fs.readFile(spec.reviews, "utf8")) as EvolutionReview[]) : [];
      const { filename, review } = await reviewUpgradeCandidate({
        artifact,
        baseline,
        candidate: JSON.parse(candidate),
        reviews,
      });
      await fs.writeFile(path.join(output, filename), JSON.stringify(review, null, 2));
      if (review.blockers.length)
        throw new Error(`Refactor required in ${node.directory}: ${review.blockers.join("; ")}`);
    }
  },
  async snapshot(root) {
    const commit = await snapshotCommit(root);
    if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit)) throw new Error("Publication did not resolve an exact commit");
    await command(root, "git", ["diff", "--exit-code", "--no-ext-diff", "--no-textconv", commit, "--"]);
    const tracked = new Set((await command(root, "git", ["ls-tree", "-r", "--name-only", "-z", commit])).split("\0"));
    for (const file of (await command(root, "git", ["ls-files", "--others", "--exclude-standard", "-z"]))
      .split("\0")
      .filter(Boolean))
      if (!tracked.has(file)) throw new Error(`Uncaptured source file ${file}`);
    return commit;
  },
  async publish(root, commit) {
    const ref = `refs/tags/quixos-reachability/${commit}`;
    const remote = await command(root, "git", ["ls-remote", "--refs", "origin", ref]);
    if (remote && remote !== `${commit}\t${ref}`) throw new Error("Immutable publication ref conflict");
    if (!remote) await command(root, "git", ["push", "origin", `${commit}:${ref}`]);
    if ((await command(root, "git", ["ls-remote", "--refs", "origin", ref])) !== `${commit}\t${ref}`)
      throw new Error("Publication response uncertain; retry the same journal");
  },
};

/** Explicit --publish only. Append-only remote retention; never moves the
 * workspace branch, activates code, or rolls back previously published nodes. */
export const applyPinUpgrades = async (
  plan: UpgradePlan,
  journalId?: string,
  implementation: UpgradeEffects = effects,
  options: { acceptEdits?: boolean } = {},
) => {
  if (implementation === effects && !plan.spec.baseline && !plan.spec.bootstrap)
    throw new Error("Publication requires an active checked baseline or explicit bootstrap:true");
  const { digest, ...body } = plan;
  if (contentDigest(body) !== digest) throw new Error("Upgrade plan digest mismatch");
  const directory = path.join(plan.workbench, ".quixos", "upgrades");
  await fs.mkdir(directory, { recursive: true, mode: 0o700 });
  if ((await fs.realpath(directory)) !== directory) throw new Error("Upgrade journals must not cross symlinks");
  const id = journalId ?? randomUUID();
  if (!/^[a-f0-9-]{36}$/.test(id)) throw new Error("Invalid upgrade journal ID");
  const filename = path.join(directory, `${id}.json`);
  return withFileLock(path.join(directory, "writer.lock"), async () => {
    try {
      const journal: Journal = journalId
        ? JSON.parse(await fs.readFile(filename, "utf8"))
        : { schemaVersion: 1, plan, steps: [] };
      if (journal.plan.digest !== plan.digest) throw new Error("Upgrade journal belongs to another plan");
      if (!journalId) await writeJournal(filename, journal);
      for (const node of plan.nodes) {
        const root = await location(plan.workbench, node.directory);
        if ((await command(root, "git", ["config", "--get", "remote.origin.url"])) !== node.source.repository)
          throw new Error("Upgrade remote changed after planning");
        let step = journal.steps.find((entry) => entry.directory === node.directory);
        if (step?.phase === "published") continue;
        if (!step) {
          if ((await treeDigest(root)) !== node.treeDigest) throw new Error(`Stale upgrade plan: ${node.directory}`);
          const files: StructuralRequest["files"] = [];
          for (const file of node.lockFiles) {
            const parsed = parseQuixosLockDocument(await fs.readFile(path.join(root, file), "utf8"));
            if (!parsed.ok) throw new Error("Invalid lock during upgrade");
            const edits = [];
            for (const resource of parsed.document.resources) {
              const dependency = plan.nodes.find((entry) => sourceKey(entry) === sourceKey(resource));
              const published =
                dependency &&
                journal.steps.find((entry) => entry.directory === dependency.directory && entry.phase === "published");
              if (published?.commit && published.commit !== resource.source.commit)
                edits.push({
                  operation: "dependency" as const,
                  kind: resource.kind,
                  name: resource.binding,
                  source: { ...resource.source, commit: published.commit },
                });
            }
            if (parsed.document.kind === "root" && plan.spec.quixos)
              edits.push({ operation: "quixos-pin" as const, source: plan.spec.quixos });
            if (edits.length) files.push({ file, edits });
          }
          const structuralPlan = files.length
            ? await planStructure(
                root,
                { kind: node.kind, source: node.source, files },
                process.env.QUIXOS_SNAPSHOT_MAP,
              )
            : undefined;
          step = {
            directory: node.directory,
            phase: "editing",
            treeDigest: node.treeDigest,
            structuralPlan,
            structuralJournal: structuralPlan ? randomUUID() : undefined,
          };
          journal.steps.push(step);
          await writeJournal(filename, journal);
        }
        if (step.phase === "editing") {
          if (step.structuralPlan) await applyStructure(step.structuralPlan, step.structuralJournal);
          else if ((await treeDigest(root)) !== step.treeDigest)
            throw new Error("Source changed before upgrade editing");
          step.treeDigest = await treeDigest(root);
          step.phase = "prepared";
          await writeJournal(filename, journal);
        }
        if (step.phase === "refactor") {
          const current = await treeDigest(root);
          if (current !== step.treeDigest && !options.acceptEdits)
            throw new Error("Refactored source requires --accept-edits when resuming");
          step.treeDigest = current;
          step.phase = "prepared";
          await writeJournal(filename, journal);
        }
        if ((await treeDigest(root)) !== step.treeDigest)
          throw new Error(`Source changed during upgrade: ${node.directory}; inspect ${filename}`);
        if (step.phase === "prepared") {
          try {
            await implementation.check(
              node,
              root,
              path.join(directory, `${id}-${node.directory.replaceAll("/", "-")}-${randomUUID()}`),
              plan.spec,
            );
          } catch (error) {
            step.phase = "refactor";
            await writeJournal(filename, journal);
            throw error;
          }
          if ((await treeDigest(root)) !== step.treeDigest) throw new Error("Source changed while checking");
          step.phase = "checked";
          await writeJournal(filename, journal);
        }
        if (step.phase === "checked") {
          step.commit = await implementation.snapshot(root);
          if ((await treeDigest(root)) !== step.treeDigest)
            throw new Error("Publication snapshot changed checked files");
          step.phase = "publishing";
          await writeJournal(filename, journal);
        }
        await implementation.publish(root, step.commit!);
        step.phase = "published";
        await writeJournal(filename, journal);
      }
      // Keep subsequent automatic upgrades associated with the newly published
      // identities, without renaming repositories or changing any selected branch.
      // Explicit-spec callers without a managed graph retain the journal as their
      // source of revisions instead.
      const graphFile = path.join(plan.workbench, ".quixos/resource-graph.json");
      let graphText: string | undefined;
      try {
        graphText = await fs.readFile(graphFile, "utf8");
      } catch (error) {
        if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
      }
      if (graphText !== undefined) {
        const previous = JSON.parse(graphText);
        const snapshots = await Promise.all(
          previous.resources.map(async (entry: { kind: string; source: Source; directory: string }) => ({
            kind: entry.kind,
            ...entry.source,
            directory: await location(
              plan.workbench,
              path.relative(plan.workbench, path.resolve(plan.workbench, entry.directory)),
            ),
          })),
        );
        for (const node of plan.nodes.filter((entry) => entry.kind !== "workspace")) {
          const step = journal.steps.find((entry) => entry.directory === node.directory)!;
          if ((await treeDigest(await location(plan.workbench, node.directory))) !== step.treeDigest)
            throw new Error("Published source changed before workbench graph refresh");
          snapshots.push({
            kind: node.kind,
            repository: node.source.repository,
            commit: step.commit,
            directory: path.join(plan.workbench, node.directory),
          });
        }
        const unique = [
          ...new Map(
            snapshots.map((entry: { kind: string; repository: string; commit: string }) => [
              JSON.stringify([entry.kind, entry.repository, entry.commit]),
              entry,
            ]),
          ).values(),
        ];
        const snapshotMap = path.join(directory, `${id}-published-snapshots.json`);
        await fs.writeFile(snapshotMap, JSON.stringify({ resources: unique }));
        const resolveResource = await createGitCapabilityResolver({
          checkoutRoot: path.join(directory, `${id}-graph-resources`),
          snapshotMap,
          snapshotOnly: true,
        });
        const rootNode = plan.nodes.find((node) => node.kind === "workspace")!;
        if (
          (await treeDigest(await location(plan.workbench, rootNode.directory))) !==
          journal.steps.find((step) => step.directory === rootNode.directory)!.treeDigest
        )
          throw new Error("Root source changed before workbench graph refresh");
        const compiled = await compileWorkspaceRepository({
          rootDirectory: await location(plan.workbench, rootNode.directory),
          resolveResource,
        });
        // Managed repositories need not currently be reachable from the workspace.
        // Keep them discoverable/checkpointed until explicitly removed by the user.
        const resources = new Map<string, ResolvedCapabilityResource>(
          compiled.resources.map((node) => [node.key, node]),
        );
        for (const node of plan.nodes.filter((entry) => entry.kind !== "workspace")) {
          const step = journal.steps.find((entry) => entry.directory === node.directory)!;
          const source = { resolver: "git" as const, repository: node.source.repository, commit: step.commit! };
          const key = `${node.kind}\0${source.repository}\0${source.commit}`;
          if (resources.has(key)) continue;
          const directory = await location(plan.workbench, node.directory);
          const standalone = await compileCapabilityResourceRepository({
            rootDirectory: directory,
            kind: node.kind as "package" | "interface" | "build",
            source,
            resolveResource,
          });
          for (const dependency of standalone.resources) resources.set(dependency.key, dependency);
          resources.set(key, {
            key,
            kind: node.kind as "package" | "interface" | "build",
            source,
            directory,
            lock: standalone.lock,
            resource: standalone.resource,
            dependencies: standalone.directResources,
          });
        }
        await writeJournal(graphFile, {
          formatVersion: 1,
          quixos: compiled.lock.quixos,
          directResources: [...compiled.directResources.entries()].map(([bindingKey, node]) => {
            const [kind, binding] = bindingKey.split("\0");
            return { kind, binding, resourceKey: node.key, directory: node.directory };
          }),
          resources: [...resources.values()].map((node) => ({
            key: node.key,
            kind: node.kind,
            source: node.source,
            directory: node.directory,
            ...(node.resource.kind === "build"
              ? {}
              : {
                  resourceId:
                    node.resource.kind === "interface"
                      ? node.resource.revision.interfaceId
                      : node.resource.revision.packageId,
                  revisionId: node.resource.revision.revisionId,
                }),
            dependencies: [...node.dependencies.entries()].map(([binding, dependency]) => ({
              binding,
              resourceKey: dependency.key,
            })),
          })),
        });
      }
      return {
        id,
        journal: filename,
        revisions: journal.steps.map((step) => ({ directory: step.directory, commit: step.commit })),
        activated: false,
      };
    } catch (error) {
      throw new Error(`${error instanceof Error ? error.message : String(error)}; upgrade journal ${filename}`, {
        cause: error,
      });
    }
  });
};