diff --git a/README.md b/README.md index c71eedc..b68095e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Shared protobuf APIs for Camino, orch, package runtimes/descriptors and values, plus the capability language/compiler. Commands below are for backend development; workspace authors use `qx-workspace check` for all verification. +The flake's default Package and check use the same `default.nix` constructor. +It declares implementation, tests, grammars, protobufs, installed Nix helpers and +the package/Yarn/TypeScript configuration as inputs. All existing compiler tests, +generated-code comparisons and emitted fixtures remain in that build; +documentation edits preserve its identity. The monorepo's maintained +`checks.x86_64-linux.quixos-protocol-source` check builds that checked compiler +before evaluating source-input mutations against the actual constructor. + ## Capability language The grammar is `grammar/QuixosCapability.g4`. The parser lowers source into the diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..56c4d34 --- /dev/null +++ b/default.nix @@ -0,0 +1,146 @@ +{ pkgs, nixpkgsNarHash }: +let + nodejs = pkgs.nodejs_24; +in +(pkgs.callPackage ./yarn-project.nix { inherit nodejs; }) { + src = pkgs.lib.fileset.toSource { + root = ./.; + fileset = pkgs.lib.fileset.unions [ + ./src + ./test + ./grammar + ./proto + ./nix + ./package.json + ./yarn.lock + ./.yarnrc.yml + ./.yarn/plugins + ./tsconfig.json + ./tsconfig.build.json + ./tsconfig.test.json + ]; + }; + overrideAttrs = old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ + pkgs.diffutils + pkgs.esbuild + pkgs.protobuf + pkgs.git + pkgs.jujutsu + pkgs.gnutar + pkgs.util-linux + pkgs.nix + ]; + buildPhase = '' + runHook preBuild + mkdir -p "$TMPDIR/generated-before" + cp --recursive src/gen "$TMPDIR/generated-before/proto" + cp --recursive src/capability-language/generated "$TMPDIR/generated-before/capability" + cp --recursive src/resource-lock/generated "$TMPDIR/generated-before/lock" + export QUIXOS_PROTO_PATH="${pkgs.protobuf}/include:$PWD/proto''${QUIXOS_PROTO_PATH:+:$QUIXOS_PROTO_PATH}" + patchShebangs node_modules/.bin node_modules/@bufbuild/protoc-gen-es/bin + yarn build + esbuild dist/src/execution-world/index.js --bundle --platform=node --target=node24 --format=esm --outfile=execution-world.mjs + esbuild dist/src/react-build/package-queries.js --bundle --platform=node --target=node24 --format=esm --outfile=react-query-generator.mjs + esbuild dist/src/bindings/client.js --bundle --platform=node --target=node24 --format=esm --outfile=client-codegen.mjs + diff --recursive --unified "$TMPDIR/generated-before/proto" src/gen + diff --recursive --unified "$TMPDIR/generated-before/capability" src/capability-language/generated + diff --recursive --unified "$TMPDIR/generated-before/lock" src/resource-lock/generated + esbuild dist/src/descriptor-check.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-descriptor-check.mjs + esbuild dist/src/capability-language/cli.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-capability-compile.mjs + esbuild dist/src/capability-language/workspace-cli.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-workspace-compile.mjs + esbuild dist/src/capability-language/resource-cli.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-resource-compile.mjs + esbuild dist/src/resource-lock/cli.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-lock-check.mjs + esbuild dist/src/bindings/cli.js --bundle --platform=node --target=node24 --format=esm --outfile=quixos-codegen-ts.mjs + esbuild dist/src/bindings/portable-client-cli.js --bundle --platform=node --target=node24 --format=esm --outfile=quixos-codegen-client.mjs + esbuild dist/src/bindings/react-platform.js --bundle --platform=node --target=node24 --format=esm --outfile=react-platform.mjs + esbuild dist/src/capability-language/tool-cli.js --bundle --platform=node --target=node24 --format=esm --outfile=quixos-qx.mjs + esbuild dist/src/capability-language/source-check-driver.js --bundle --platform=node --target=node24 --format=esm --outfile=source-check-driver.mjs + runHook postBuild + ''; + doCheck = true; + checkPhase = '' + runHook preCheck + yarn test + node dist/test/emit-portable-reader.js + node dist/test/emit-managed.js + runHook postCheck + ''; + installPhase = '' + runHook preInstall + mkdir -p "$out" + install -Dm644 nix/checked-package.nix "$out/share/checked-package.nix" + install -Dm644 nix/checked-artifacts.nix "$out/share/checked-artifacts.nix" + install -Dm644 nix/package-context.nix "$out/share/package-context.nix" + install -Dm644 nix/lock-source.nix "$out/share/lock-source.nix" + install -Dm644 nix/package-context.mjs "$out/share/package-context.mjs" + install -Dm644 nix/check-package.mjs "$out/share/check-package.mjs" + install -Dm644 nix/check-execution.mjs "$out/share/check-execution.mjs" + substitute nix/checked-candidate.nix "$out/share/checked-candidate.nix" \ + --replace-fail '@nixpkgs@' '${pkgs.path}' \ + --replace-fail '@nixpkgsNarHash@' '${nixpkgsNarHash}' + cp --reflink=auto --recursive grammar "$out/grammar" + cp --reflink=auto --recursive proto "$out/proto" + cp --reflink=auto --recursive dist "$out/dist" + cp package.json "$out/package.json" + mkdir -p "$out/share/quixos-protocol/qualification" + cp -r dist/portable-reader dist/managed "$out/share/quixos-protocol/qualification/" + mkdir -p "$out/bin" + for tool in quixos-codegen-ts quixos-codegen-client quixos-qx; do + printf '#!/bin/sh\nexec ${pkgs.nodejs_24}/bin/node "%s/libexec/quixos-protocol/%s.mjs" "$@"\n' "$out" "$tool" > "$out/bin/$tool" + chmod +x "$out/bin/$tool" + done + cat > "$out/bin/quixos-descriptor-check" < "$out/bin/quixos-capability-compile" < "$out/bin/quixos-workspace-compile" < "$out/bin/quixos-resource-compile" < "$out/bin/quixos-lock-check" < "$out/bin/$tool" - chmod +x "$out/bin/$tool" - done - cat > "$out/bin/quixos-descriptor-check" < "$out/bin/quixos-capability-compile" < "$out/bin/quixos-workspace-compile" < "$out/bin/quixos-resource-compile" < "$out/bin/quixos-lock-check" < JSON.stringify(value).replaceAll("${", "\\${"); +async function writable(directory) { + await fs.chmod(directory, 0o700); + for (const item of await fs.readdir(directory, { withFileTypes: true })) { + const file = path.join(directory, item.name); + if (item.isDirectory()) await writable(file); + else { + const info = await fs.stat(file); + await fs.chmod(file, info.mode & 0o111 ? 0o700 : 0o600); + } + } +} +function identity() { + const expression = `let + pkgs = import ${nixString(config.nixpkgs)} { system = ${nixString(config.system)}; }; + compiler = import (${nixString(source)} + "/default.nix") { + inherit pkgs; + nixpkgsNarHash = ${nixString(config.nixpkgsNarHash)}; + }; + in { + derivation = compiler.drvPath; + yarn = compiler.yarn-freestanding.drvPath; + source = toString compiler.src; + }`; + const result = spawnSync( + "nix", + [ + "--store", + "local?root=" + process.env.QUIXOS_PROTOCOL_EVALUATION_STORE, + "eval", + "--offline", + "--impure", + "--json", + "--expr", + expression, + ], + { encoding: "utf8", timeout: 30000, maxBuffer: 1048576 }, + ); + assert.equal(result.status, 0, result.error?.message ?? result.stderr); + return JSON.parse(result.stdout); +} +async function edit(file, mutate, check) { + const before = await fs.readFile(file, "utf8"); + try { + await fs.writeFile(file, mutate(before)); + await check(); + } finally { + await fs.writeFile(file, before); + } +} +const cases = []; +async function record(name, check) { + await check(); + cases.push(name); + console.log("PROTOCOL_SOURCE_INPUT: " + name); +} +try { + await fs.cp(config.source, source, { recursive: true }); + await writable(source); + const baseline = identity(); + await record("real production compiler and generated Yarn launcher identities", async () => { + assert.equal(baseline.derivation, config.expected); + assert.equal(baseline.yarn, config.yarn); + }); + for (const file of ["README.md", "flake.nix", "default.nix", "yarn-project.nix"]) { + await record(file + " commentary preserves compiler identity", () => + edit( + path.join(source, file), + (text) => text + (file.endsWith(".nix") ? "\n# source qualification\n" : "\nSource qualification.\n"), + async () => assert.deepEqual(identity(), baseline), + ), + ); + } + await record("new documentation preserves compiler identity", async () => { + const file = path.join(source, "NOTES.md"); + try { + await fs.writeFile(file, "qualification\n"); + assert.deepEqual(identity(), baseline); + } finally { + await fs.rm(file, { force: true }); + } + }); + async function compilerOnly() { + const changed = identity(); + assert.notEqual(changed.derivation, baseline.derivation); + assert.equal(changed.yarn, baseline.yarn); + } + for (const file of [ + "src/capability-language/cli.ts", + "test/emit-managed.ts", + "grammar/QuixosCapability.g4", + "proto/quixos/refs.proto", + "nix/check-package.mjs", + "package.json", + "yarn.lock", + ".yarnrc.yml", + ".yarn/plugins/yarn-plugin-nixify.cjs", + "tsconfig.json", + "tsconfig.build.json", + "tsconfig.test.json", + ]) { + await record(file + " remains a compiler input", () => + edit(path.join(source, file), (text) => text + "\n", compilerOnly), + ); + } + for (const file of ["src/future.ts", "test/future.ts", "nix/future.nix"]) { + await record(file + " is included without a file inventory update", async () => { + const target = path.join(source, file); + try { + await fs.writeFile(target, "qualification\n"); + await compilerOnly(); + } finally { + await fs.rm(target, { force: true }); + } + }); + } + await record("executable recipe changes invalidate compiler identity", () => + edit( + path.join(source, "default.nix"), + (text) => { + assert(text.includes("runHook preBuild")); + return text.replace("runHook preBuild", "runHook preBuild\n echo qualification"); + }, + compilerOnly, + ), + ); + await record("selected generated archive pin remains a compiler input", () => + edit( + path.join(source, "yarn-project.nix"), + (text) => { + const changedHash = "sha512-" + Buffer.alloc(64, 0x42).toString("base64"); + const pattern = /("@types\/node@npm:[^\n]+hash = ")[^"]+(";)/; + assert(pattern.test(text)); + return text.replace(pattern, (_, before, after) => before + changedHash + after); + }, + async () => { + await compilerOnly(); + assert.equal(identity().source, baseline.source); + }, + ), + ); + await record("installed Nixpkgs trust identity remains a compiler input", async () => { + const previous = config.nixpkgsNarHash; + try { + config.nixpkgsNarHash = "sha256-" + Buffer.alloc(32, 0x42).toString("base64"); + assert.notEqual(config.nixpkgsNarHash, previous); + await compilerOnly(); + assert.equal(identity().source, baseline.source); + } finally { + config.nixpkgsNarHash = previous; + } + }); + assert.deepEqual(identity(), baseline, "Every private mutation was restored"); + await fs.mkdir(output, { recursive: true }); + await fs.writeFile( + path.join(output, "protocol-source.json"), + JSON.stringify( + { + codec: "quixos-protocol-source-qualification/1", + ...baseline, + cases, + nestedBuilds: false, + guestExecution: false, + }, + null, + 2, + ) + "\n", + ); +} finally { + await fs.rm(scratch, { recursive: true, force: true }); +} diff --git a/source-qualification.nix b/source-qualification.nix new file mode 100644 index 0000000..cef70f3 --- /dev/null +++ b/source-qualification.nix @@ -0,0 +1,73 @@ +{ + pkgs, + library, + nixpkgsNarHash, +}: +let + comparisonIdentity = import ../tools/local-first/derivation-identity.nix; + source = pkgs.lib.fileset.toSource { + root = ./.; + fileset = pkgs.lib.fileset.unions [ + ./src + ./test + ./grammar + ./proto + ./nix + ./package.json + ./yarn.lock + ./.yarnrc.yml + ./.yarn/plugins + ./tsconfig.json + ./tsconfig.build.json + ./tsconfig.test.json + ./default.nix + ./yarn-project.nix + ./flake.nix + ./README.md + ]; + }; + inputs = pkgs.writeText "protocol-source-inputs.json" ( + builtins.toJSON { + inherit source nixpkgsNarHash; + nixpkgs = pkgs.path; + system = pkgs.stdenv.hostPlatform.system; + expected = comparisonIdentity library; + yarn = comparisonIdentity library.yarn-freestanding; + } + ); + closure = pkgs.closureInfo { rootPaths = [ pkgs.path ]; }; +in +pkgs.runCommand "quixos-protocol-source-check" + { + nativeBuildInputs = [ + pkgs.nodejs_24 + pkgs.nix + ]; + passthru.ciDiagnosticArtifacts."protocol-source.json" = { + file = "protocol-source.json"; + maxBytes = 8192; + }; + } + '' + # The ordinary compiler still owns all tests and emitted fixtures. Finish + # that same checked build before the new identity-only evaluations. + test -e ${library} + export HOME="$TMPDIR/home" + export NIX_STATE_DIR="$TMPDIR/nix-state" + export NIX_LOG_DIR="$TMPDIR/nix-log" + export NIX_CONF_DIR="$TMPDIR/nix-conf" + export NIX_USER_CONF_FILES=/dev/null + export NIX_CONFIG='experimental-features = nix-command flakes + substituters = + builders = + max-jobs = 0 + allow-import-from-derivation = false + accept-flake-config = false' + export QUIXOS_PROTOCOL_EVALUATION_STORE="$TMPDIR/evaluation-store" + mkdir -p "$HOME" "$NIX_STATE_DIR" "$NIX_LOG_DIR" "$NIX_CONF_DIR" "$QUIXOS_PROTOCOL_EVALUATION_STORE/nix/store" + while IFS= read -r dependency; do + cp -a --no-preserve=ownership "$dependency" "$QUIXOS_PROTOCOL_EVALUATION_STORE$dependency" + done < ${closure}/store-paths + nix-store --store "local?root=$QUIXOS_PROTOCOL_EVALUATION_STORE" --load-db < ${closure}/registration + timeout --kill-after=5s 180s node ${./source-qualification.mjs} ${inputs} "$out" + ''