From 1f7aa801301995649ce1d8cfab32e4fb61f9fb07 Mon Sep 17 00:00:00 2001 From: "Timothy J. Aveni" Date: Thu, 1 Oct 2026 01:37:46 -0700 Subject: [PATCH] Instrument and improve incremental workspace checks Profile source capture, Nix evaluation/realization, resource/query checking, and implementation builds without putting timings into immutable outputs. Keep modular repositories; bound independent capture work, snapshot jj once per observation, and retain final concurrent-edit and Git completeness checks. Compose exact resource/closure/frontend/toolchain checks as memoized Nix derivations. Preserve whole-workspace validation and the full compiler path when no checked dependency is supplied. Share canonical Nixpkgs by content, index workspace build contexts once, and narrow lock/driver/compiler inputs with conservative complete-source fallback. Batch React witness checks only when distinct source closures cannot share ambient declarations; otherwise run the original independent TypeScript checks. Seed local authoring toolchains with exact canonical fetch identities when they match the selected inputs; retain store-path inputs for overrides. Add generic correctness checks rather than permanent Reader timing/content fixtures. Verify actual Reader edits and rejection/recovery in a private workbench, preserving the original user's workbench and runtime. Query checking currently caches at resource/closure boundaries, not individual query semantic dependency boundaries. Template helper pins and reproducible fixture snapshots follow in a separate publication change. --- PORTABLE_ARTIFACTS.md | 10 +++++ artifacts.nix | 20 +++++++-- assemble-package.nix | 9 +++- build-artifacts.mjs | 73 ++++++++++++++++-------------- build-components.mjs | 92 ++++++++++++++++++-------------------- build-profile.mjs | 12 +++++ build-tools.nix | 13 ++++++ component-type-batch.mjs | 49 ++++++++++++++++++++ component-types.mjs | 51 +++++++++++++++++++++ component-types.test.mjs | 59 ++++++++++++++++++++++++ package-set.test.nix | 57 +++++++++++++++++++++++ quixos-package-helpers.nix | 34 ++++++++++++++ react-contracts.nix | 3 +- 13 files changed, 393 insertions(+), 89 deletions(-) create mode 100644 build-profile.mjs create mode 100644 build-tools.nix create mode 100644 component-type-batch.mjs create mode 100644 component-types.mjs create mode 100644 component-types.test.mjs create mode 100644 package-set.test.nix diff --git a/PORTABLE_ARTIFACTS.md b/PORTABLE_ARTIFACTS.md index 55a35a3..b285675 100644 --- a/PORTABLE_ARTIFACTS.md +++ b/PORTABLE_ARTIFACTS.md @@ -31,6 +31,16 @@ excluded from the implementation source tree. The `plan` argument on the low-lev helper is a composition convenience for fixtures; it is projected before building. Package recipes receive the contract directly and can be checked without a workspace. +Inside `checkedArtifacts = context: ...`, the package helpers expose +`camino.packageSet { inherit nixpkgs system context; }` as the usual package set +selection. It shares the workspace composition's evaluated Nixpkgs when the +source content, system, and default import arguments match. Pass ordinary Nixpkgs +arguments such as `config` or `overlays` to use a separate import. A direct +`import nixpkgs { inherit system; }` also works, with additional evaluation cost. +The implementation groups depend on the complete execution compiler bundle and +JavaScript build drivers by content; unrelated checker CLI and publication +metadata changes do not enter those inputs. + The handler entry exports `registry = createRegistry(handlers)` from the generated bindings. One registry contains many exports. The builder always supplies a server implementation for the generated process service. A service-only package instead diff --git a/artifacts.nix b/artifacts.nix index dabb747..f5ca613 100644 --- a/artifacts.nix +++ b/artifacts.nix @@ -34,7 +34,13 @@ group ? false, }: let - compiler = "${protocol}/libexec/quixos-protocol/execution-world.mjs"; + buildTools = import ./build-tools.nix; + # The standalone bundle is the complete code dependency of this driver. + # Changes to unrelated checker CLIs must not invalidate every package group. + compiler = builtins.path { + path = "${protocol}/libexec/quixos-protocol/execution-world.mjs"; + name = "execution-world.mjs"; + }; config = pkgs.writeText "camino-artifact-build.json" ( builtins.toJSON { inherit @@ -92,9 +98,15 @@ let serviceAdapter = ./portable-service.mjs; } ); - compiled = pkgs.runCommand "camino-compiled-artifacts" { passthru.buildConfig = config; } '' - ${pkgs.nodejs_24}/bin/node ${./.}/build-artifacts.mjs ${config} - ''; + compiled = + pkgs.runCommand "camino-compiled-artifacts" + { + passthru.buildConfig = config; + QUIXOS_TRACE_PROFILE = "1"; + } + '' + ${pkgs.nodejs_24}/bin/node ${buildTools}/build-artifacts.mjs ${config} + ''; in if group then compiled diff --git a/assemble-package.nix b/assemble-package.nix index 87e4e5d..f4f4835 100644 --- a/assemble-package.nix +++ b/assemble-package.nix @@ -4,7 +4,12 @@ artifacts, }: let + buildTools = import ./build-tools.nix; groups = map (artifact: artifact.caminoGroup or artifact) artifacts; + compiler = builtins.path { + path = "${context.generator}/libexec/quixos-protocol/execution-world.mjs"; + name = "execution-world.mjs"; + }; config = pkgs.writeText "camino-package-assembly.json" ( builtins.toJSON { inherit groups; @@ -14,7 +19,7 @@ let packageRevisionId sharedRuntime ; - compiler = "${context.generator}/libexec/quixos-protocol/execution-world.mjs"; + inherit compiler; node = "${pkgs.nodejs_24}/bin/node"; esbuild = "${pkgs.esbuild}/bin/esbuild"; shell = "${pkgs.runtimeShell}"; @@ -30,5 +35,5 @@ pkgs.runCommand "camino-package" }; } '' - ${pkgs.nodejs_24}/bin/node ${./.}/assemble-package.mjs ${config} + ${pkgs.nodejs_24}/bin/node ${buildTools}/assemble-package.mjs ${config} '' diff --git a/build-artifacts.mjs b/build-artifacts.mjs index f1bc3c1..040e3b3 100644 --- a/build-artifacts.mjs +++ b/build-artifacts.mjs @@ -1,4 +1,5 @@ import { installDependencies } from "./install-dependencies.mjs"; +import { measureBuild } from "./build-profile.mjs"; import { buildService } from "./build-service.mjs"; import { buildComponents } from "./build-components.mjs"; /** Trusted Nix build driver. No package-defined build hooks run between generation and witnessing. */ @@ -102,46 +103,50 @@ for (const { target, mode, settings } of builds) { source, ); const entry = relative(settings.entries?.[target] ?? settings.entry); - execFileSync( - config.node, - [ - config.tsc, - "--noEmit", - "--strict", - "--skipLibCheck", - "false", - "--target", - "es2024", - "--lib", - "es2024,dom,esnext.disposable", - "--module", - "nodenext", - "--moduleResolution", - "nodenext", - entry, - ], - { cwd: work, stdio: "inherit" }, + measureBuild("portable-typecheck", { package: pkg.revision, target, mode }, () => + execFileSync( + config.node, + [ + config.tsc, + "--noEmit", + "--strict", + "--skipLibCheck", + "false", + "--target", + "es2024", + "--lib", + "es2024,dom,esnext.disposable", + "--module", + "nodenext", + "--moduleResolution", + "nodenext", + entry, + ], + { cwd: work, stdio: "inherit" }, + ), ); const bundlePath = mode === "remote" ? "share/quixos/remote/server/module.mjs" : `share/quixos/portable/${target}/module.mjs`; await fs.mkdir(path.dirname(path.join(output, bundlePath)), { recursive: true }); const coreImport = target === "browser" ? coreURL : coreServer; const metadata = path.join(work, "bundle-meta.json"); - execFileSync( - config.esbuild, - [ - entry, - "--bundle", - `--platform=${target === "browser" ? "browser" : "node"}`, - "--format=esm", - "--minify-whitespace", - "--target=es2024", - `--outfile=${path.join(output, bundlePath)}`, - `--metafile=${metadata}`, - `--alias:@quixos/camino-replica-core=${coreImport}`, - `--external:${coreImport}`, - ], - { cwd: work, stdio: "inherit" }, + measureBuild("portable-bundle", { package: pkg.revision, target, mode }, () => + execFileSync( + config.esbuild, + [ + entry, + "--bundle", + `--platform=${target === "browser" ? "browser" : "node"}`, + "--format=esm", + "--minify-whitespace", + "--target=es2024", + `--outfile=${path.join(output, bundlePath)}`, + `--metafile=${metadata}`, + `--alias:@quixos/camino-replica-core=${coreImport}`, + `--external:${coreImport}`, + ], + { cwd: work, stdio: "inherit" }, + ), ); const meta = JSON.parse(await fs.readFile(metadata, "utf8")); const built = Object.values(meta.outputs).find((o) => o.entryPoint); diff --git a/build-components.mjs b/build-components.mjs index d87a825..535594f 100644 --- a/build-components.mjs +++ b/build-components.mjs @@ -1,4 +1,6 @@ import { installDependencies } from "./install-dependencies.mjs"; +import { checkComponentTypes } from "./component-types.mjs"; +import { measureBuild } from "./build-profile.mjs"; import fs from "node:fs/promises"; import path from "node:path"; import { createHash } from "node:crypto"; @@ -152,16 +154,12 @@ export async function buildComponents({ config, compiler, contract, pkg, output, propsImports.join("\n") + `\nexport {};\ndeclare module ${JSON.stringify("./" + bindingPath.replace(/\.ts$/, ".js"))} { interface ModuleProps {${propsMembers.join("\n")}} }\n`, ); - const components = []; - const { checkControls } = await import(pathToFileURL(path.join(settings.sdk, "lint-controls.mjs"))); - const { default: ts } = await import(pathToFileURL(path.resolve(path.dirname(config.tsc), "../lib/typescript.js"))); - const checkedControls = new Set(); + const witnesses = []; for (const [id, entry] of entries) { const declaration = declarations.find((component) => component.id === id), source = compiler.artifactPath(entry.entry); if (!/^[$A-Z_a-z][$\w]*$/.test(entry.export)) throw Error("Invalid component export name"); - const key = digest(Buffer.from(id)).slice(0, 24), - directory = "share/quixos/components/" + key; + const key = digest(Buffer.from(id)).slice(0, 24); const witness = path.join(work, "component-check-" + key + ".tsx"); const module = pkg.modules.find((module) => module.id === id); await fs.writeFile( @@ -171,30 +169,26 @@ export async function buildComponents({ config, compiler, contract, pkg, output, ? `import type {ComponentProps} from "react";declare const props:Pick,"subject"|"input">;const mount = ;\n` : ""), ); - execFileSync( - config.node, - [ - config.tsc, - "--noEmit", - "--strict", - "--skipLibCheck", - "false", - "--target", - "es2024", - "--lib", - "es2024,dom,dom.iterable,esnext.disposable", - "--module", - "nodenext", - "--moduleResolution", - "nodenext", - "--jsx", - "react-jsx", - path.basename(witness), - assetPath, - propsPath, - ], - { cwd: work, stdio: "inherit" }, - ); + witnesses.push(path.basename(witness)); + } + checkComponentTypes({ + node: config.node, + tsc: config.tsc, + work, + witnesses, + assets: assetPath, + props: propsPath, + packageRevision: pkg.revision, + }); + const components = []; + const { checkControls } = await import(pathToFileURL(path.join(settings.sdk, "lint-controls.mjs"))); + const { default: ts } = await import(pathToFileURL(path.resolve(path.dirname(config.tsc), "../lib/typescript.js"))); + const checkedControls = new Set(); + for (const [id, entry] of entries) { + const declaration = declarations.find((component) => component.id === id), + source = compiler.artifactPath(entry.entry), + key = digest(Buffer.from(id)).slice(0, 24), + directory = "share/quixos/components/" + key; const metadata = path.join(work, "component-meta-" + key + ".json"); const facade = "component-entry-" + key + ".ts"; await fs.writeFile( @@ -202,24 +196,26 @@ export async function buildComponents({ config, compiler, contract, pkg, output, `export {${entry.export}} from ${JSON.stringify("./" + source.replace(/\.[cm]?tsx?$/, ".js"))};\n`, ); await fs.mkdir(path.join(output, directory), { recursive: true }); - execFileSync( - config.esbuild, - [ - facade, - "--bundle", - "--platform=browser", - "--format=esm", - "--target=es2024", - "--jsx=automatic", - '--define:process.env.NODE_ENV="production"', - "--metafile=" + metadata, - "--outfile=" + path.join(output, directory, "module.mjs"), - ...sharedAliases, - ...["svg", "png", "jpg", "jpeg", "gif", "webp", "avif", "woff", "woff2", "ttf", "ico"].map( - (extension) => "--loader:." + extension + "=dataurl", - ), - ], - { cwd: work, stdio: "inherit" }, + measureBuild("component-bundle", { package: pkg.revision, component: id }, () => + execFileSync( + config.esbuild, + [ + facade, + "--bundle", + "--platform=browser", + "--format=esm", + "--target=es2024", + "--jsx=automatic", + '--define:process.env.NODE_ENV="production"', + "--metafile=" + metadata, + "--outfile=" + path.join(output, directory, "module.mjs"), + ...sharedAliases, + ...["svg", "png", "jpg", "jpeg", "gif", "webp", "avif", "woff", "woff2", "ttf", "ico"].map( + (extension) => "--loader:." + extension + "=dataurl", + ), + ], + { cwd: work, stdio: "inherit" }, + ), ); const meta = JSON.parse(await fs.readFile(metadata, "utf8")), outputs = Object.entries(meta.outputs); diff --git a/build-profile.mjs b/build-profile.mjs new file mode 100644 index 0000000..4391102 --- /dev/null +++ b/build-profile.mjs @@ -0,0 +1,12 @@ +/** Timing is log data, never part of an artifact or its witness. */ +export function measureBuild(phase, details, action) { + const started = performance.now(); + try { + return action(); + } finally { + if (process.env.QUIXOS_TRACE_PROFILE === "1") + console.error( + `[qx-build-profile] ${JSON.stringify({ phase, ms: Math.round(performance.now() - started), ...details })}`, + ); + } +} diff --git a/build-tools.nix b/build-tools.nix new file mode 100644 index 0000000..c80795d --- /dev/null +++ b/build-tools.nix @@ -0,0 +1,13 @@ +# Standalone build drivers import sibling JavaScript modules. Their code is an +# input; Nix helper APIs and test sources do not belong to that dependency. +builtins.path { + path = ./.; + name = "camino-build-tools"; + filter = + filename: type: + let + name = builtins.baseNameOf filename; + in + type == "directory" + || (builtins.match ".*\\.mjs" name != null && builtins.match ".*\\.test\\.mjs" name == null); +} diff --git a/component-type-batch.mjs b/component-type-batch.mjs new file mode 100644 index 0000000..4ce801a --- /dev/null +++ b/component-type-batch.mjs @@ -0,0 +1,49 @@ +import path from "node:path"; +import { createRequire } from "node:module"; + +const [tsc, count, ...args] = process.argv.slice(2); +const ts = createRequire(import.meta.url)(path.resolve(path.dirname(tsc), "../lib/typescript.js")); +const parsed = ts.parseCommandLine(args); +const witnesses = parsed.fileNames.slice(0, Number(count)); +const shared = parsed.fileNames.slice(Number(count)); +const host = ts.createCompilerHost(parsed.options, true); +// These programs only discover source closures; they do not bind or check. +// Sharing immutable parsed source files avoids reading/parsing the same libs +// for every witness. Only the final combined program performs typechecking. +const sources = new Map(); +const getSourceFile = host.getSourceFile; +host.getSourceFile = (filename, ...options) => { + if (!sources.has(filename)) sources.set(filename, getSourceFile(filename, ...options)); + return sources.get(filename); +}; +for (const method of ["fileExists", "directoryExists", "getDirectories", "readFile", "realpath"]) { + if (!host[method]) continue; + const original = host[method], + cache = new Map(); + host[method] = (filename) => { + if (!cache.has(filename)) cache.set(filename, original(filename)); + return cache.get(filename); + }; +} +const closures = witnesses.map( + (witness) => new Set(ts.createProgram([witness, ...shared], parsed.options, host).getSourceFiles()), +); +const common = new Set([...closures[0]].filter((source) => closures.every((closure) => closure.has(source)))); +const ambient = (node) => + ts.isModuleDeclaration(node) || ts.isNamespaceExportDeclaration(node) || ts.forEachChild(node, ambient); +// A file that is visible to just some witnesses must not contribute globals +// or ambient module declarations to the others. Otherwise a combined success +// could hide an independent failure. Script files are conservatively unsafe. +if ( + closures.some((closure) => + [...closure].some((source) => !common.has(source) && (!ts.isExternalModule(source) || ambient(source))), + ) +) + process.exit(2); + +const program = ts.createProgram(parsed.fileNames, parsed.options, host); +const diagnostics = [...parsed.errors, ...ts.getPreEmitDiagnostics(program)]; +if (diagnostics.length) { + console.error(ts.formatDiagnosticsWithColorAndContext(diagnostics, host)); + process.exit(1); +} diff --git a/component-types.mjs b/component-types.mjs new file mode 100644 index 0000000..8da07ce --- /dev/null +++ b/component-types.mjs @@ -0,0 +1,51 @@ +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { measureBuild } from "./build-profile.mjs"; + +// Component witnesses share bindings and usually the same source graph. +// Check a bounded batch once when distinct source closures cannot supply +// ambient declarations to one another. Otherwise use the original checks. +export function checkComponentTypes({ node, tsc, work, witnesses, assets, props, packageRevision }) { + const checkTypes = (files, combined = false) => + execFileSync( + node, + [ + ...(combined ? [fileURLToPath(new URL("./component-type-batch.mjs", import.meta.url))] : []), + tsc, + ...(combined ? [String(files.length)] : []), + "--noEmit", + "--strict", + "--skipLibCheck", + "false", + "--target", + "es2024", + "--lib", + "es2024,dom,dom.iterable,esnext.disposable", + "--module", + "nodenext", + "--moduleResolution", + "nodenext", + "--jsx", + "react-jsx", + ...files, + assets, + props, + ], + { cwd: work, stdio: combined ? "pipe" : "inherit" }, + ); + measureBuild("component-typecheck", { package: packageRevision, components: witnesses.length }, () => { + for (let start = 0; start < witnesses.length; start += 1000) { + const batch = witnesses.slice(start, start + 1000); + if (batch.length === 1) checkTypes(batch); + else { + try { + checkTypes(batch, true); + } catch (error) { + if (!(error.status > 0) && error.code !== "ENOBUFS") throw error; + console.error("Component batch requires independent witness checks"); + for (const witness of batch) checkTypes([witness]); + } + } + } + }); +} diff --git a/component-types.test.mjs b/component-types.test.mjs new file mode 100644 index 0000000..3bfce73 --- /dev/null +++ b/component-types.test.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { checkComponentTypes } from "./component-types.mjs"; + +const tsc = process.env.QUIXOS_TYPESCRIPT; +if (!tsc) throw Error("Pass the selected TypeScript compiler path"); +async function fixture(context, left, right) { + const work = await mkdtemp(path.join(os.tmpdir(), "component-types-")); + context.after(() => rm(work, { recursive: true, force: true })); + await Promise.all( + Object.entries({ "left.ts": left, "right.ts": right, "assets.d.ts": "", "props.d.ts": "" }).map(([file, source]) => + writeFile(path.join(work, file), source), + ), + ); + return { + node: process.execPath, + tsc, + work, + witnesses: ["left.ts", "right.ts"], + assets: "assets.d.ts", + props: "props.d.ts", + packageRevision: "test", + }; +} +test("shared component typechecking accepts all valid witnesses", async (context) => { + checkComponentTypes(await fixture(context, "export const left: string = 'ok';", "export const right: number = 1;")); +}); +test("independently valid ambient declarations use the full individual check fallback", async (context) => { + checkComponentTypes( + await fixture( + context, + "export {}; declare global { interface Application { value: string; } }", + "export {}; declare global { interface Application { value: number; } }", + ), + ); +}); +test("a type error in any witness still rejects the artifact", async (context) => { + const input = await fixture(context, "export const left: string = 'ok';", "export const right: string = 1;"); + assert.throws(() => checkComponentTypes(input)); +}); +test("one witness's globals cannot make an independently invalid witness pass", async (context) => { + const input = await fixture( + context, + "export {}; declare global { interface Application { value: string; } }", + "export const right: Application = {value: 'ok'};", + ); + assert.throws(() => checkComponentTypes(input)); +}); +test("an ambient module from another source closure cannot resolve a missing import", async (context) => { + const input = await fixture( + context, + "declare module 'missing-component-package' { export const value: string; }", + "import {value} from 'missing-component-package'; export const right = value;", + ); + assert.throws(() => checkComponentTypes(input)); +}); diff --git a/package-set.test.nix b/package-set.test.nix new file mode 100644 index 0000000..034e57b --- /dev/null +++ b/package-set.test.nix @@ -0,0 +1,57 @@ +{ + pkgs, + nixpkgs, + system, +}: +let + packageSet = (import ./quixos-package-helpers.nix).packageSet; + shared = { + marker = "shared"; + }; + environment = { + inherit system; + nixpkgsSource = toString nixpkgs; + pkgs = shared; + narHash = nixpkgs.narHash; + }; + standard = { + inherit nixpkgs system; + context.buildEnvironment = environment; + }; + fallback = packageSet { inherit nixpkgs system; }; +in +assert packageSet standard == shared; +assert + packageSet ( + standard + // { + context.buildEnvironment = environment // { + nixpkgsSource = "same-content-under-another-name"; + }; + } + ) == shared; +assert + (packageSet ( + standard + // { + context.buildEnvironment = environment // { + system = "another-system"; + }; + } + )).path == fallback.path; +assert + (packageSet ( + standard + // { + context.buildEnvironment = environment // { + nixpkgsSource = "another-source"; + narHash = "another-hash"; + }; + } + )).path == fallback.path; +assert + (packageSet (standard // { overlays = [ (_: _: { marker = "overlay"; }) ]; })).marker == "overlay"; +assert (packageSet (standard // { config.allowBroken = true; })).config.allowBroken; +pkgs.runCommand "camino-package-set-check" { } '' + touch "$out" +'' diff --git a/quixos-package-helpers.nix b/quixos-package-helpers.nix index 356a7d4..8f3ecef 100644 --- a/quixos-package-helpers.nix +++ b/quixos-package-helpers.nix @@ -847,6 +847,40 @@ let in { mkCaminoReactPackage = import ./react-package.nix; + # Share the composition's package set only for the same exact Nixpkgs source, + # system and default import arguments. Config/overlay/cross-build choices use + # ordinary Nixpkgs evaluation. No package implementation is substituted. + packageSet = + args@{ + nixpkgs, + system, + context ? { }, + ... + }: + let + authoredArgs = builtins.removeAttrs args [ + "nixpkgs" + "context" + ]; + importArgs = { + config = { }; + overlays = [ ]; + } + // authoredArgs; + environment = context.buildEnvironment or null; + in + if + builtins.attrNames authoredArgs == [ "system" ] + && environment != null + && environment.system == system + && ( + environment.nixpkgsSource == toString nixpkgs + || ((nixpkgs.narHash or null) != null && nixpkgs.narHash == (environment.narHash or null)) + ) + then + environment.pkgs + else + import nixpkgs importArgs; mkCaminoReactContracts = import ./react-contracts.nix; mkCaminoYarnDependencies = import ./yarn-dependencies.nix; mkCaminoYarnShell = import ./yarn-shell.nix; diff --git a/react-contracts.nix b/react-contracts.nix index a037d3c..fcc1e40 100644 --- a/react-contracts.nix +++ b/react-contracts.nix @@ -6,6 +6,7 @@ dependencies ? null, }: let + buildTools = import ./build-tools.nix; config = pkgs.writeText "camino-react-contracts.json" ( builtins.toJSON { source = import ./implementation-source.nix { @@ -33,5 +34,5 @@ let ); in pkgs.runCommand "camino-react-contracts" { } '' - ${pkgs.nodejs_24}/bin/node ${./.}/build-react-contracts.mjs ${config} + ${pkgs.nodejs_24}/bin/node ${buildTools}/build-react-contracts.mjs ${config} ''