quixos-protocol/src/capability-language/assembly.ts

import { readFile, realpath } from "node:fs/promises";
import path from "node:path";
import { profile } from "./profile.js";
import { loadQxSources, resolveQxSources } from "./source-loader.js";
import { compileQuery, compileRepositoryQuery } from "../query/compile.js";
import { queryTreeProjections } from "../query/tree.js";
import type { QueryDeclaration } from "../query/types.js";
import { compileQueryComposition, type QueryComposition } from "../query/composition.js";
import { checkQueryTemplate } from "../query/templates.js";
import { readRepositorySource } from "./source-loader.js";
import { parseControllerCatalog } from "../capability-model/controllers.js";
import { linkQueries } from "../query/link.js";
import {
  capabilityId,
  compileWorkspaceRevision,
  validateMigrationCatalog,
  contentDigest,
  type AtomDefinition,
  type CompiledWorkspaceRevision,
  type InterfaceRevision,
  type PackageRevision,
  type SourceRevision,
  type WorkspaceRevision,
} from "../capability-model/index.js";
import {
  loadQuixosLock,
  type GitSource,
  type LockedResource,
  type QuixosRepositoryLock,
} from "../resource-lock/index.js";
import {
  compileCapabilityResourceSource,
  compileCapabilitySource,
  type CapabilityImportEnvironment,
  type CapabilityResource,
  type CapabilityResourceImport,
} from "./parser.js";

export type CapabilityRepositorySnapshot = {
  directory: string;
};

export type CapabilityRepositoryResolver = (
  source: GitSource,
  kind: LockedResource["kind"],
) => Promise<CapabilityRepositorySnapshot>;

/** Already checked outputs of this compiler, composed by the Nix build DAG.
 * Source capture/check reports never grant this authority. */
export type CheckedResourceResolver = (
  kind: LockedResource["kind"],
  source: GitSource,
  lock: QuixosRepositoryLock,
  dependencies: ReadonlyMap<string, ResolvedCapabilityResource>,
) => Promise<CapabilityResource | undefined>;

/** An explicitly selected build frontend supplies ordinary query sources. The
 * assembler never chooses a frontend by inspecting an interface/contract ID.
 * Generated sources go through the same query and imported-fragment checks. */
export type PackageQueryGenerator = ((input: {
  readonly package: Readonly<PackageRevision>;
  readonly interfaces: readonly InterfaceRevision[];
  readonly readSource: (name: string) => Promise<string>;
}) => Promise<{
  readonly queries: readonly QueryDeclaration[];
  readonly documents: ReadonlyMap<string, string>;
  readonly compositions?: readonly QueryComposition[];
  readonly artifacts?: Readonly<Record<string, unknown>>;
}>) & {
  /** Optional explicit frontend stage after workspace binding resolution.
   * Its output still passes normal workspace/query validation. */
  completeWorkspace?: (workspace: WorkspaceRevision) => Promise<void>;
};

export type ResolvedCapabilityResource = {
  key: string;
  kind: LockedResource["kind"];
  source: GitSource;
  directory: string;
  lock: QuixosRepositoryLock;
  resource: CapabilityResource | { kind: "build" };
  dependencies: ReadonlyMap<string, ResolvedCapabilityResource>;
};

export type CompiledWorkspaceRepository = {
  workspace: WorkspaceRevision;
  plan: CompiledWorkspaceRevision;
  lock: QuixosRepositoryLock;
  resources: ResolvedCapabilityResource[];
  capabilityResources: ResolvedCapabilityResource[];
  directResources: ReadonlyMap<string, ResolvedCapabilityResource>;
};

export type CompiledCapabilityResourceRepository = {
  resource: CapabilityResource | { kind: "build" };
  lock: QuixosRepositoryLock;
  resources: ResolvedCapabilityResource[];
  capabilityResources: ResolvedCapabilityResource[];
  directResources: ReadonlyMap<string, ResolvedCapabilityResource>;
};

const sourceKey = (kind: LockedResource["kind"], source: GitSource) =>
  `${kind}\0${source.repository}\0${source.commit.toLowerCase()}`;

const bindingKey = (kind: LockedResource["kind"], binding: string) => `${kind}\0${binding}`;

const importsKey = (entry: CapabilityResourceImport | LockedResource) => bindingKey(entry.kind, entry.binding);

const assertImportsMatchLock = (
  label: string,
  imports: readonly CapabilityResourceImport[],
  resources: readonly LockedResource[],
) => {
  const authored = [...imports].map(importsKey).sort();
  const locked = resources
    .filter((entry) => entry.kind !== "build")
    .map(importsKey)
    .sort();
  if (authored.length !== locked.length || authored.some((entry, index) => entry !== locked[index])) {
    throw new Error(
      `${label} imports do not match quixos.lock:\n` +
        `authored: ${authored.join(", ") || "none"}\n` +
        `locked: ${locked.join(", ") || "none"}`,
    );
  }
};

const exactRevisions = <
  Revision extends {
    revisionId: string;
    source: SourceRevision;
  },
>(
  revisions: readonly Revision[],
) => {
  const seen = new Set<string>();
  return revisions.filter((revision) => {
    const key = `${revision.revisionId}\0${revision.source.repository}\0${revision.source.commit}`;
    if (seen.has(key)) return false;
    seen.add(key);
    return true;
  });
};

const exactAtoms = (atoms: readonly AtomDefinition[]) => {
  const seen = new Set<string>();
  return atoms.filter((atom) => {
    if (seen.has(atom.id)) return false;
    seen.add(atom.id);
    return true;
  });
};

const diagnosticsMessage = (
  label: string,
  diagnostics: readonly {
    fileName: string;
    line: number;
    column: number;
    phase: string;
    code: string;
    message: string;
    path?: string;
  }[],
) =>
  `${label} did not compile:\n${diagnostics
    .map((entry) => {
      const location =
        entry.line > 0
          ? `${entry.fileName}:${entry.line}:${entry.column + 1}`
          : `${entry.fileName}${entry.path ? `:${entry.path}` : ""}`;
      return `${location}: ${entry.phase} ${entry.code}: ${entry.message}`;
    })
    .join("\n")}`;

const revisionFor = (node: ResolvedCapabilityResource) => {
  if (node.resource.kind === "build") throw Error("Build sources have no capability revision");
  return node.resource.revision;
};

const resourceClosure = (
  roots: readonly ResolvedCapabilityResource[],
  capabilityOnly = false,
): ResolvedCapabilityResource[] => {
  const result: ResolvedCapabilityResource[] = [];
  const seen = new Set<string>();
  const visit = (node: ResolvedCapabilityResource) => {
    if (seen.has(node.key) || (capabilityOnly && node.kind === "build")) return;
    seen.add(node.key);
    for (const dependency of node.dependencies.values()) visit(dependency);
    result.push(node);
  };
  for (const root of roots) visit(root);
  return result;
};

const environmentFor = (
  direct: readonly [LockedResource, ResolvedCapabilityResource][],
  closure: readonly ResolvedCapabilityResource[],
): CapabilityImportEnvironment => {
  closure = resourceClosure(
    direct.map(([, node]) => node),
    true,
  );
  return {
    interfaces: new Map(
      direct.flatMap(([locked, node]) =>
        node.resource.kind === "interface" ? [[locked.binding, node.resource.revision] as const] : [],
      ),
    ),
    packages: new Map(
      direct.flatMap(([locked, node]) =>
        node.resource.kind === "package" ? [[locked.binding, node.resource.revision] as const] : [],
      ),
    ),
    interfaceClosure: exactRevisions(
      closure.flatMap((node) => [
        ...(node.resource.kind === "interface" ? [node.resource.revision] : []),
        ...(node.resource.kind === "build" ? [] : (node.resource.specializations ?? [])),
      ]),
    ),
    packageClosure: exactRevisions(
      closure.flatMap((node) => (node.resource.kind === "package" ? [node.resource.revision] : [])),
    ),
    externalAtoms: exactAtoms(
      closure.flatMap((node) => (node.resource.kind === "build" ? [] : node.resource.externalAtoms)),
    ),
  };
};

const createResourceGraphResolver = (
  quixosCommit: string,
  resolveResource: CapabilityRepositoryResolver,
  generateQueries?: PackageQueryGenerator,
  checkedResource?: CheckedResourceResolver,
) => {
  const graphStarted = performance.now();
  let checkedResources = 0,
    reusedResources = 0,
    checkedQueries = 0;
  const resolved = new Map<string, Promise<ResolvedCapabilityResource>>();
  const active: string[] = [];

  const visit = async (
    locked: LockedResource,
    suppliedSnapshot?: CapabilityRepositorySnapshot,
  ): Promise<ResolvedCapabilityResource> => {
    const key = sourceKey(locked.kind, locked.source);
    const cached = resolved.get(key);
    if (cached) {
      if (active.includes(key)) {
        throw new Error(`Resource dependency cycle: ${[...active, key].join(" -> ")}`);
      }
      return await cached;
    }
    const pending = (async () => {
      active.push(key);
      try {
        const snapshot = suppliedSnapshot ?? (await resolveResource(locked.source, locked.kind));
        const lockResult = await loadQuixosLock(path.join(snapshot.directory, "quixos.lock"));
        if (!lockResult.ok) {
          throw new Error(diagnosticsMessage(`${locked.kind} ${locked.binding} lock`, lockResult.diagnostics));
        }
        if (lockResult.lock.quixos.policy) {
          throw new Error(
            `${locked.kind} ${locked.binding} lock declares workspace Quixos policy ` +
              `${lockResult.lock.quixos.policy}; resource locks may only declare their exact authored-against commit`,
          );
        }
        if (lockResult.lock.quixos.commit.toLowerCase() !== quixosCommit.toLowerCase()) {
          throw new Error(
            `${locked.kind} ${locked.binding} selects Quixos ${lockResult.lock.quixos.commit}, ` +
              `but the repository graph selects ${quixosCommit}`,
          );
        }
        const directPairs: Array<[LockedResource, ResolvedCapabilityResource]> = [];
        for (const dependency of lockResult.lock.resources) {
          directPairs.push([dependency, await visit(dependency)]);
        }
        if (locked.kind === "build") {
          return {
            key,
            kind: locked.kind,
            source: locked.source,
            directory: snapshot.directory,
            lock: lockResult.lock,
            resource: { kind: "build" as const },
            dependencies: new Map(
              directPairs.map(([dependency, node]) => [bindingKey(dependency.kind, dependency.binding), node]),
            ),
          };
        }
        const dependencyClosure = resourceClosure(directPairs.map(([, node]) => node));
        const dependencies = new Map(
          directPairs.map(([dependency, node]) => [bindingKey(dependency.kind, dependency.binding), node]),
        );
        const checked = await checkedResource?.(locked.kind, locked.source, lockResult.lock, dependencies);
        if (checked) {
          if (
            checked.kind !== locked.kind ||
            checked.revision.source.repository !== locked.source.repository ||
            checked.revision.source.commit !== locked.source.commit
          )
            throw new Error("Checked resource identity mismatch");
          assertImportsMatchLock("Checked resource", checked.imports, lockResult.lock.resources);
          reusedResources++;
          return {
            key,
            kind: locked.kind,
            source: locked.source,
            directory: snapshot.directory,
            lock: lockResult.lock,
            resource: structuredClone(checked),
            dependencies,
          };
        }
        const checkStarted = performance.now();
        const environment = environmentFor(directPairs, dependencyClosure);
        const manifestName = locked.kind === "interface" ? "interface.qx" : "package.qx";
        const manifestPath = path.join(snapshot.directory, manifestName);
        const sourceText = await readFile(manifestPath, "utf8");
        const compiled = compileCapabilityResourceSource(sourceText, {
          source: locked.source,
          fileName: manifestPath,
          environment,
        });
        if (!compiled.ok) {
          throw new Error(diagnosticsMessage(`${locked.kind} ${locked.binding}`, compiled.diagnostics));
        }
        if (compiled.resource.kind !== locked.kind) {
          throw new Error(`${manifestPath} declares ${compiled.resource.kind}, not ${locked.kind}`);
        }
        assertImportsMatchLock(manifestPath, compiled.resource.imports, lockResult.lock.resources);
        if (compiled.resource.kind === "package") {
          const queryInterfaces = [
            ...(environment.interfaceClosure ?? []),
            ...(compiled.resource.specializations ?? []),
          ];
          const generated = await generateQueries?.({
            package: structuredClone(compiled.resource.revision),
            interfaces: structuredClone(queryInterfaces),
            readSource: (name) => readRepositorySource(snapshot.directory, name),
          });
          if (generated) {
            if (generated.artifacts) compiled.resource.revision.buildArtifacts = structuredClone(generated.artifacts);
            const queries = [...(compiled.resource.revision.queries ?? [])];
            for (const query of generated.queries) {
              if (queries.some((entry) => entry.id === query.id || entry.displayName === query.displayName))
                throw new Error(`Generated query conflicts with an authored query: ${query.id}`);
              if (!generated.documents.has(query.document))
                throw new Error(`Generated query has no source: ${query.document}`);
              queries.push(structuredClone(query));
            }
            compiled.resource.revision.queries = queries;
          }
          for (const template of compiled.resource.revision.queryTemplates ?? [])
            await checkQueryTemplate(template, queryInterfaces, (name) =>
              readRepositorySource(snapshot.directory, name),
            );
          if (compiled.resource.revision.queries?.length) {
            const pkg = compiled.resource.revision;
            const checked = new Map<string, import("../query/types.js").CheckedQuery>();
            const compile = async (
              id: string,
              parents: readonly string[] = [],
            ): Promise<import("../query/types.js").CheckedQuery> => {
              if (parents.includes(id)) throw new Error(`Query fragment cycle: ${[...parents, id].join(" -> ")}`);
              const previous = checked.get(id);
              if (previous) return previous;
              const query = pkg.queries!.find((query) => query.id === id);
              if (!query) throw new Error(`Unknown fragment ${pkg.revisionId}/${id}`);
              const resolveFragment = async (packageId: string, fragmentId: string) => {
                if (packageId === pkg.revisionId) return compile(fragmentId, [...parents, id]);
                const imported = environment.packageClosure
                  ?.find((other) => other.revisionId === packageId)
                  ?.checkedQueries?.find((query) => query.declaration.id === fragmentId);
                if (!imported) throw new Error(`Unchecked imported fragment ${packageId}/${fragmentId}`);
                return imported;
              };
              const result = generated?.documents.has(query.document)
                ? await compileQuery(
                    query,
                    queryInterfaces,
                    async (name) =>
                      generated.documents.get(name) ?? (await readRepositorySource(snapshot.directory, name)),
                    resolveFragment,
                  )
                : await compileRepositoryQuery(snapshot.directory, query, queryInterfaces, resolveFragment);
              checked.set(id, result);
              return result;
            };
            pkg.checkedQueries = [];
            for (const query of pkg.queries!) pkg.checkedQueries.push(await compile(query.id));
            const projections = pkg.checkedQueries.flatMap(queryTreeProjections);
            for (const projection of projections) {
              if (pkg.queries!.some((query) => query.id === projection.declaration.id))
                throw new Error(`Generated Query Tree projection conflicts with ${projection.declaration.id}`);
              pkg.queries!.push(projection.declaration);
              pkg.checkedQueries.push(projection);
            }
          }
          if (generated?.compositions?.length) {
            const pkg = compiled.resource.revision;
            const queries = new Map((pkg.checkedQueries ?? []).map((query) => [query.declaration.id, query]));
            for (const dependency of environment.packageClosure ?? [])
              for (const query of dependency.checkedQueries ?? [])
                queries.set(JSON.stringify([dependency.revisionId, query.declaration.id]), query);
            const ids = new Set<string>();
            pkg.queryCompositions = generated.compositions.map((composition) => {
              if (ids.has(composition.id)) throw new Error(`Duplicate query composition: ${composition.id}`);
              ids.add(composition.id);
              return compileQueryComposition(composition, queries);
            });
          }
          let controllerText: string | undefined;
          try {
            controllerText = await readRepositorySource(snapshot.directory, "quixos.controllers.json");
          } catch (error) {
            if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
          }
          if (controllerText !== undefined)
            compiled.resource.revision.controllers = parseControllerCatalog(
              controllerText,
              queryInterfaces,
              compiled.resource.revision.checkedQueries,
            );
          let catalogText: string | undefined;
          try {
            catalogText = await readFile(path.join(snapshot.directory, "quixos.migrations.json"), "utf8");
          } catch (error) {
            if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
          }
          if (catalogText !== undefined) {
            const catalog = validateMigrationCatalog(
              JSON.parse(catalogText),
              new Set(compiled.resource.revision.exports.map((entry) => entry.id)),
            );
            const root = await realpath(snapshot.directory);
            for (const migration of catalog.migrations) {
              const implementation = await realpath(path.join(root, migration.implementation.file));
              if (!implementation.startsWith(`${root}${path.sep}`))
                throw new Error("Migration implementation escapes its package");
              if (contentDigest(await readFile(implementation, "utf8")) !== migration.implementation.digest)
                throw new Error(`Migration implementation digest mismatch: ${migration.id}`);
            }
            compiled.resource.revision.migrationCatalog = catalog;
          }
        }
        checkedResources++;
        const queries =
          compiled.resource.kind === "package" ? (compiled.resource.revision.checkedQueries?.length ?? 0) : 0;
        checkedQueries += queries;
        profile("resource-check", checkStarted, {
          kind: locked.kind,
          resource: compiled.resource.revision.displayName,
          queries,
        });
        return {
          key,
          kind: locked.kind,
          source: locked.source,
          directory: snapshot.directory,
          lock: lockResult.lock,
          resource: compiled.resource,
          dependencies: new Map(
            directPairs.map(([dependency, node]) => [bindingKey(dependency.kind, dependency.binding), node]),
          ),
        };
      } finally {
        active.pop();
      }
    })();
    resolved.set(key, pending);
    return await pending;
  };

  return {
    visit,
    nodes: async () => {
      const nodes = await Promise.all(resolved.values());
      profile("resource-graph", graphStarted, { checkedResources, reusedResources, checkedQueries });
      return nodes;
    },
  };
};

export const compileCapabilityResourceRepository = async (options: {
  rootDirectory: string;
  kind: LockedResource["kind"];
  source: GitSource;
  resolveResource: CapabilityRepositoryResolver;
  generateQueries?: PackageQueryGenerator;
  checkedResource?: CheckedResourceResolver;
}): Promise<CompiledCapabilityResourceRepository> => {
  const lockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock"));
  if (!lockResult.ok) {
    throw new Error(diagnosticsMessage("Resource lock", lockResult.diagnostics));
  }
  const resolver = createResourceGraphResolver(
    lockResult.lock.quixos.commit,
    options.resolveResource,
    options.generateQueries,
    options.checkedResource,
  );
  const root = await resolver.visit(
    {
      kind: options.kind,
      binding: "<root>",
      source: options.source,
    },
    { directory: options.rootDirectory },
  );
  return {
    resource: root.resource,
    lock: root.lock,
    resources: await resolver.nodes(),
    capabilityResources: resourceClosure([root], true),
    directResources: root.dependencies,
  };
};

export const compileWorkspaceRepository = async (options: {
  rootDirectory: string;
  resolveResource: CapabilityRepositoryResolver;
  workspaceId?: string;
  workspaceRevisionId?: string;
  sourceRootCommit?: string;
  /** An editor's proposed source snapshot; locks and dependency revisions remain exact. */
  readSource?: (name: string) => Promise<string>;
  generateQueries?: PackageQueryGenerator;
  checkedResource?: CheckedResourceResolver;
}): Promise<CompiledWorkspaceRepository> => {
  const rootLockResult = await loadQuixosLock(path.join(options.rootDirectory, "quixos.lock"));
  if (!rootLockResult.ok) {
    throw new Error(diagnosticsMessage("Workspace lock", rootLockResult.diagnostics));
  }
  const rootLock = rootLockResult.lock;
  const resolver = createResourceGraphResolver(
    rootLock.quixos.commit,
    options.resolveResource,
    options.generateQueries,
    options.checkedResource,
  );

  const directPairs: Array<[LockedResource, ResolvedCapabilityResource]> = [];
  for (const locked of rootLock.resources) {
    directPairs.push([locked, await resolver.visit(locked)]);
  }
  const nodes = await resolver.nodes();
  const environment = environmentFor(directPairs, nodes);
  const workspacePath = path.join(options.rootDirectory, "workspace.qx");
  const sources = options.readSource
    ? await resolveQxSources(options.readSource)
    : await loadQxSources(options.rootDirectory);
  const compiled = compileCapabilitySource(sources.source, workspacePath, environment);
  if (!compiled.ok) {
    throw new Error(
      diagnosticsMessage(
        "Workspace",
        compiled.diagnostics.map((diagnostic) =>
          diagnostic.line > 0
            ? { ...diagnostic, ...sources.originalPosition(diagnostic.line, diagnostic.column) }
            : diagnostic,
        ),
      ),
    );
  }
  assertImportsMatchLock(workspacePath, compiled.imports, rootLock.resources);
  const availableInterfaceIds = new Set((environment.interfaceClosure ?? []).map((iface) => iface.revisionId));
  const availableAtomIds = new Set(compiled.workspace.atoms.map((atom) => atom.id));
  for (const node of resourceClosure(
    directPairs.map(([, node]) => node),
    true,
  )) {
    if (node.resource.kind === "build") continue;
    for (const requirement of node.resource.externalInterfaces) {
      if (!availableInterfaceIds.has(requirement.revisionId)) {
        throw new Error(
          `${node.kind} ${node.resource.revision.displayName} requires external interface ` +
            `${requirement.binding} (${requirement.revisionId}), but the workspace resource graph does not provide it`,
        );
      }
    }
    for (const atom of node.resource.externalAtoms) {
      if (!availableAtomIds.has(atom.id)) {
        throw new Error(
          `${node.kind} ${node.resource.revision.displayName} requires external atom ` +
            `${atom.displayName} (${atom.id}), but the workspace does not define it`,
        );
      }
    }
  }
  const workspace: WorkspaceRevision = {
    ...compiled.workspace,
    ...(options.workspaceId ? { workspaceId: capabilityId.workspace(options.workspaceId) } : {}),
    ...(options.workspaceRevisionId
      ? { id: capabilityId.workspaceRevision(options.workspaceRevisionId) }
      : options.sourceRootCommit
        ? {
            id: capabilityId.workspaceRevision(
              `workspace-revision:${options.workspaceId ?? compiled.workspace.workspaceId}:${options.sourceRootCommit}`,
            ),
          }
        : {}),
    ...(options.sourceRootCommit ? { sourceRootCommit: options.sourceRootCommit } : {}),
  };
  await options.generateQueries?.completeWorkspace?.(workspace);
  const checked = compileWorkspaceRevision(workspace);
  if (!checked.ok) {
    throw new Error(
      `Instantiated workspace did not compile:\n${checked.issues
        .map((entry) => `${entry.path}: ${entry.message}`)
        .join("\n")}`,
    );
  }
  const linkedQueries = linkQueries(workspace);
  if (linkedQueries.length) {
    workspace.linkedQueries = linkedQueries;
    checked.plan.source.linkedQueries = structuredClone(linkedQueries);
  }
  return {
    workspace,
    plan: checked.plan,
    lock: rootLock,
    resources: nodes,
    capabilityResources: resourceClosure(
      directPairs.map(([, node]) => node),
      true,
    ),
    directResources: new Map(directPairs.map(([locked, node]) => [bindingKey(locked.kind, locked.binding), node])),
  };
};