diff --git a/.quixos-subtree-source.json b/.quixos-subtree-source.json index 7b8f75e..79a0f22 100644 --- a/.quixos-subtree-source.json +++ b/.quixos-subtree-source.json @@ -1,7 +1,7 @@ { "version": 1, "sourceRepo": "https://gitea.internal.quixos.org/quixos/quixos.git", - "sourceCommit": "473886a58fdacf15f2c873b77da96faf3be7fa65", + "sourceCommit": "dd5a29c267f0313d46ac845e5d341cde4c67e4d1", "sourcePath": "quixos-protocol", "exportName": "quixos-protocol", "mirrorRemote": "https://repos.quixos.org/quixos-toolchain/quixos-protocol.git" diff --git a/README.md b/README.md index c71eedc..b68095e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Shared protobuf APIs for Camino, orch, package runtimes/descriptors and values, plus the capability language/compiler. Commands below are for backend development; workspace authors use `qx-workspace check` for all verification. +The flake's default Package and check use the same `default.nix` constructor. +It declares implementation, tests, grammars, protobufs, installed Nix helpers and +the package/Yarn/TypeScript configuration as inputs. All existing compiler tests, +generated-code comparisons and emitted fixtures remain in that build; +documentation edits preserve its identity. The monorepo's maintained +`checks.x86_64-linux.quixos-protocol-source` check builds that checked compiler +before evaluating source-input mutations against the actual constructor. + ## Capability language The grammar is `grammar/QuixosCapability.g4`. The parser lowers source into the diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..56c4d34 --- /dev/null +++ b/default.nix @@ -0,0 +1,146 @@ +{ pkgs, nixpkgsNarHash }: +let + nodejs = pkgs.nodejs_24; +in +(pkgs.callPackage ./yarn-project.nix { inherit nodejs; }) { + src = pkgs.lib.fileset.toSource { + root = ./.; + fileset = pkgs.lib.fileset.unions [ + ./src + ./test + ./grammar + ./proto + ./nix + ./package.json + ./yarn.lock + ./.yarnrc.yml + ./.yarn/plugins + ./tsconfig.json + ./tsconfig.build.json + ./tsconfig.test.json + ]; + }; + overrideAttrs = old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ + pkgs.diffutils + pkgs.esbuild + pkgs.protobuf + pkgs.git + pkgs.jujutsu + pkgs.gnutar + pkgs.util-linux + pkgs.nix + ]; + buildPhase = '' + runHook preBuild + mkdir -p "$TMPDIR/generated-before" + cp --recursive src/gen "$TMPDIR/generated-before/proto" + cp --recursive src/capability-language/generated "$TMPDIR/generated-before/capability" + cp --recursive src/resource-lock/generated "$TMPDIR/generated-before/lock" + export QUIXOS_PROTO_PATH="${pkgs.protobuf}/include:$PWD/proto''${QUIXOS_PROTO_PATH:+:$QUIXOS_PROTO_PATH}" + patchShebangs node_modules/.bin node_modules/@bufbuild/protoc-gen-es/bin + yarn build + esbuild dist/src/execution-world/index.js --bundle --platform=node --target=node24 --format=esm --outfile=execution-world.mjs + esbuild dist/src/react-build/package-queries.js --bundle --platform=node --target=node24 --format=esm --outfile=react-query-generator.mjs + esbuild dist/src/bindings/client.js --bundle --platform=node --target=node24 --format=esm --outfile=client-codegen.mjs + diff --recursive --unified "$TMPDIR/generated-before/proto" src/gen + diff --recursive --unified "$TMPDIR/generated-before/capability" src/capability-language/generated + diff --recursive --unified "$TMPDIR/generated-before/lock" src/resource-lock/generated + esbuild dist/src/descriptor-check.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-descriptor-check.mjs + esbuild dist/src/capability-language/cli.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-capability-compile.mjs + esbuild dist/src/capability-language/workspace-cli.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-workspace-compile.mjs + esbuild dist/src/capability-language/resource-cli.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-resource-compile.mjs + esbuild dist/src/resource-lock/cli.js \ + --bundle --platform=node --target=node24 --format=esm \ + --outfile=quixos-lock-check.mjs + esbuild dist/src/bindings/cli.js --bundle --platform=node --target=node24 --format=esm --outfile=quixos-codegen-ts.mjs + esbuild dist/src/bindings/portable-client-cli.js --bundle --platform=node --target=node24 --format=esm --outfile=quixos-codegen-client.mjs + esbuild dist/src/bindings/react-platform.js --bundle --platform=node --target=node24 --format=esm --outfile=react-platform.mjs + esbuild dist/src/capability-language/tool-cli.js --bundle --platform=node --target=node24 --format=esm --outfile=quixos-qx.mjs + esbuild dist/src/capability-language/source-check-driver.js --bundle --platform=node --target=node24 --format=esm --outfile=source-check-driver.mjs + runHook postBuild + ''; + doCheck = true; + checkPhase = '' + runHook preCheck + yarn test + node dist/test/emit-portable-reader.js + node dist/test/emit-managed.js + runHook postCheck + ''; + installPhase = '' + runHook preInstall + mkdir -p "$out" + install -Dm644 nix/checked-package.nix "$out/share/checked-package.nix" + install -Dm644 nix/checked-artifacts.nix "$out/share/checked-artifacts.nix" + install -Dm644 nix/package-context.nix "$out/share/package-context.nix" + install -Dm644 nix/lock-source.nix "$out/share/lock-source.nix" + install -Dm644 nix/package-context.mjs "$out/share/package-context.mjs" + install -Dm644 nix/check-package.mjs "$out/share/check-package.mjs" + install -Dm644 nix/check-execution.mjs "$out/share/check-execution.mjs" + substitute nix/checked-candidate.nix "$out/share/checked-candidate.nix" \ + --replace-fail '@nixpkgs@' '${pkgs.path}' \ + --replace-fail '@nixpkgsNarHash@' '${nixpkgsNarHash}' + cp --reflink=auto --recursive grammar "$out/grammar" + cp --reflink=auto --recursive proto "$out/proto" + cp --reflink=auto --recursive dist "$out/dist" + cp package.json "$out/package.json" + mkdir -p "$out/share/quixos-protocol/qualification" + cp -r dist/portable-reader dist/managed "$out/share/quixos-protocol/qualification/" + mkdir -p "$out/bin" + for tool in quixos-codegen-ts quixos-codegen-client quixos-qx; do + printf '#!/bin/sh\nexec ${pkgs.nodejs_24}/bin/node "%s/libexec/quixos-protocol/%s.mjs" "$@"\n' "$out" "$tool" > "$out/bin/$tool" + chmod +x "$out/bin/$tool" + done + cat > "$out/bin/quixos-descriptor-check" < "$out/bin/quixos-capability-compile" < "$out/bin/quixos-workspace-compile" < "$out/bin/quixos-resource-compile" < "$out/bin/quixos-lock-check" < "$out/bin/$tool" - chmod +x "$out/bin/$tool" - done - cat > "$out/bin/quixos-descriptor-check" < "$out/bin/quixos-capability-compile" < "$out/bin/quixos-workspace-compile" < "$out/bin/quixos-resource-compile" < "$out/bin/quixos-lock-check" < JSON.stringify(value).replaceAll("${", "\\${"); +async function writable(directory) { + await fs.chmod(directory, 0o700); + for (const item of await fs.readdir(directory, { withFileTypes: true })) { + const file = path.join(directory, item.name); + if (item.isDirectory()) await writable(file); + else { + const info = await fs.stat(file); + await fs.chmod(file, info.mode & 0o111 ? 0o700 : 0o600); + } + } +} +function identity() { + const expression = `let + pkgs = import (/. + ${nixString(config.nixpkgs)}) { system = ${nixString(config.system)}; }; + compiler = import ((/. + ${nixString(source)}) + "/default.nix") { + inherit pkgs; + nixpkgsNarHash = builtins.appendContext ${nixString(config.nixpkgsNarHash)} + (builtins.fromJSON ${nixString(JSON.stringify(config.narHashContext))}); + }; + in { + derivation = compiler.drvPath; + yarn = compiler.yarn-freestanding.drvPath; + source = toString compiler.src; + metadata = (import ((/. + ${nixString(source)}) + "/source-qualification-metadata.nix")) compiler; + }`; + const result = spawnSync( + "nix", + [ + "--store", + "local?root=" + process.env.QUIXOS_PROTOCOL_EVALUATION_STORE, + "eval", + "--offline", + "--impure", + "--json", + "--expr", + expression, + ], + { encoding: "utf8", timeout: 30000, maxBuffer: 1048576 }, + ); + assert.equal(result.status, 0, result.error?.message ?? result.stderr); + return JSON.parse(result.stdout); +} +function summarized(value) { + const text = JSON.stringify(value) ?? "undefined"; + return text.length < 500 + ? value + : { + bytes: Buffer.byteLength(text), + sha256: createHash("sha256").update(text).digest("hex"), + prefix: text.slice(0, 180), + }; +} +function structuralDifferences(actual, expected, field = "", result = []) { + if (JSON.stringify(actual) === JSON.stringify(expected)) return result; + if ( + Array.isArray(actual) && + Array.isArray(expected) && + [...actual, ...expected].every((x) => typeof x === "string") + ) { + result.push({ + field, + onlyActual: actual.filter((x) => !expected.includes(x)), + onlyExpected: expected.filter((x) => !actual.includes(x)), + actualCount: actual.length, + expectedCount: expected.length, + sequenceChanged: true, + }); + } else if ( + actual && + expected && + typeof actual === "object" && + typeof expected === "object" && + !Array.isArray(actual) && + !Array.isArray(expected) + ) { + for (const key of new Set([...Object.keys(actual), ...Object.keys(expected)])) + structuralDifferences(actual[key], expected[key], field ? field + "." + key : key, result); + } else { + const difference = { field, actual: summarized(actual), expected: summarized(expected) }; + if (typeof actual === "string" && typeof expected === "string" && Math.max(actual.length, expected.length) >= 500) { + let offset = 0; + while (offset < actual.length && offset < expected.length && actual[offset] === expected[offset]) offset++; + difference.offset = offset; + difference.actualExcerpt = actual.slice(Math.max(0, offset - 80), offset + 200); + difference.expectedExcerpt = expected.slice(Math.max(0, offset - 80), offset + 200); + } + result.push(difference); + } + return result; +} +function baselineDifference(baseline) { + const { metadata, ...identities } = baseline; + console.log( + "PROTOCOL_SOURCE_BASELINE: " + + JSON.stringify({ + actual: identities, + expected: config.expected, + expectedSource: config.expectedSource, + evaluatorVersion: config.evaluatorVersion, + pinnedNixVersion: config.pinnedNixVersion, + narHashContext: config.narHashContext, + }), + ); + console.log( + "PROTOCOL_SOURCE_BASELINE_DIFF: " + + JSON.stringify(structuralDifferences(metadata, config.expectedMetadata)).slice(0, 10000), + ); +} +async function edit(file, mutate, check) { + const before = await fs.readFile(file, "utf8"); + try { + await fs.writeFile(file, mutate(before)); + await check(); + } finally { + await fs.writeFile(file, before); + } +} +const cases = []; +async function record(name, check) { + await check(); + cases.push(name); + console.log("PROTOCOL_SOURCE_INPUT: " + name); +} +try { + await fs.cp(config.source, source, { recursive: true }); + await writable(source); + const baseline = identity(); + if (baseline.derivation !== config.expected) baselineDifference(baseline); + await record("real production compiler and generated Yarn launcher identities", async () => { + assert.equal(baseline.derivation, config.expected); + assert.equal(baseline.yarn, config.yarn); + }); + for (const file of ["README.md", "flake.nix", "default.nix", "yarn-project.nix"]) { + await record(file + " commentary preserves compiler identity", () => + edit( + path.join(source, file), + (text) => text + (file.endsWith(".nix") ? "\n# source qualification\n" : "\nSource qualification.\n"), + async () => assert.deepEqual(identity(), baseline), + ), + ); + } + await record("new documentation preserves compiler identity", async () => { + const file = path.join(source, "NOTES.md"); + try { + await fs.writeFile(file, "qualification\n"); + assert.deepEqual(identity(), baseline); + } finally { + await fs.rm(file, { force: true }); + } + }); + async function compilerOnly() { + const changed = identity(); + assert.notEqual(changed.derivation, baseline.derivation); + assert.equal(changed.yarn, baseline.yarn); + } + for (const file of [ + "src/capability-language/cli.ts", + "test/emit-managed.ts", + "grammar/QuixosCapability.g4", + "proto/quixos/refs.proto", + "nix/check-package.mjs", + "package.json", + "yarn.lock", + ".yarnrc.yml", + ".yarn/plugins/yarn-plugin-nixify.cjs", + "tsconfig.json", + "tsconfig.build.json", + "tsconfig.test.json", + ]) { + await record(file + " remains a compiler input", () => + edit(path.join(source, file), (text) => text + "\n", compilerOnly), + ); + } + for (const file of ["src/future.ts", "test/future.ts", "nix/future.nix"]) { + await record(file + " is included without a file inventory update", async () => { + const target = path.join(source, file); + try { + await fs.writeFile(target, "qualification\n"); + await compilerOnly(); + } finally { + await fs.rm(target, { force: true }); + } + }); + } + await record("executable recipe changes invalidate compiler identity", () => + edit( + path.join(source, "default.nix"), + (text) => { + assert(text.includes("runHook preBuild")); + return text.replace("runHook preBuild", "runHook preBuild\n echo qualification"); + }, + compilerOnly, + ), + ); + await record("selected generated archive pin remains a compiler input", () => + edit( + path.join(source, "yarn-project.nix"), + (text) => { + const changedHash = "sha512-" + Buffer.alloc(64, 0x42).toString("base64"); + const pattern = /("@types\/node@npm:[^\n]+hash = ")[^"]+(";)/; + assert(pattern.test(text)); + return text.replace(pattern, (_, before, after) => before + changedHash + after); + }, + async () => { + await compilerOnly(); + assert.equal(identity().source, baseline.source); + }, + ), + ); + await record("installed Nixpkgs trust identity remains a compiler input", async () => { + const previous = config.nixpkgsNarHash; + try { + config.nixpkgsNarHash = "sha256-" + Buffer.alloc(32, 0x42).toString("base64"); + assert.notEqual(config.nixpkgsNarHash, previous); + await compilerOnly(); + assert.equal(identity().source, baseline.source); + } finally { + config.nixpkgsNarHash = previous; + } + }); + assert.deepEqual(identity(), baseline, "Every private mutation was restored"); + await fs.mkdir(output, { recursive: true }); + await fs.writeFile( + path.join(output, "protocol-source.json"), + JSON.stringify( + { + codec: "quixos-protocol-source-qualification/1", + derivation: baseline.derivation, + yarn: baseline.yarn, + source: baseline.source, + cases, + metadataContext: config.narHashContext, + evaluatorVersion: config.evaluatorVersion, + pinnedNixVersion: config.pinnedNixVersion, + nestedBuilds: false, + guestExecution: false, + }, + null, + 2, + ) + "\n", + ); +} finally { + await fs.rm(scratch, { recursive: true, force: true }); +} diff --git a/source-qualification.nix b/source-qualification.nix new file mode 100644 index 0000000..4af2934 --- /dev/null +++ b/source-qualification.nix @@ -0,0 +1,84 @@ +{ + pkgs, + library, + nixpkgsSource, + nixpkgsNarHash, +}: +assert builtins.isString nixpkgsSource; +let + comparisonIdentity = import ../tools/local-first/derivation-identity.nix; + inspect = import ./source-qualification-metadata.nix; + source = pkgs.lib.fileset.toSource { + root = ./.; + fileset = pkgs.lib.fileset.unions [ + ./src + ./test + ./grammar + ./proto + ./nix + ./package.json + ./yarn.lock + ./.yarnrc.yml + ./.yarn/plugins + ./tsconfig.json + ./tsconfig.build.json + ./tsconfig.test.json + ./default.nix + ./source-qualification-metadata.nix + ./yarn-project.nix + ./flake.nix + ./README.md + ]; + }; + inputs = pkgs.writeText "protocol-source-inputs.json" ( + builtins.toJSON { + inherit source nixpkgsNarHash; + narHashContext = builtins.getContext nixpkgsNarHash; + evaluatorVersion = builtins.nixVersion; + pinnedNixVersion = pkgs.nix.version; + # toJSON ingests path values. Keep the original flake input's outPath + # string, so importing it and interpolating pkgs.path does not copy twice. + nixpkgs = nixpkgsSource; + system = pkgs.stdenv.hostPlatform.system; + expected = comparisonIdentity library; + expectedMetadata = inspect library; + expectedSource = builtins.unsafeDiscardStringContext (toString library.src); + yarn = comparisonIdentity library.yarn-freestanding; + } + ); + closure = pkgs.closureInfo { rootPaths = [ nixpkgsSource ]; }; +in +pkgs.runCommand "quixos-protocol-source-check" + { + nativeBuildInputs = [ + pkgs.nodejs_24 + pkgs.nix + ]; + passthru.ciDiagnosticArtifacts."protocol-source.json" = { + file = "protocol-source.json"; + maxBytes = 8192; + }; + } + '' + # The ordinary compiler still owns all tests and emitted fixtures. Finish + # that same checked build before the new identity-only evaluations. + test -e ${library} + export HOME="$TMPDIR/home" + export NIX_STATE_DIR="$TMPDIR/nix-state" + export NIX_LOG_DIR="$TMPDIR/nix-log" + export NIX_CONF_DIR="$TMPDIR/nix-conf" + export NIX_USER_CONF_FILES=/dev/null + export NIX_CONFIG='experimental-features = nix-command flakes + substituters = + builders = + max-jobs = 0 + allow-import-from-derivation = false + accept-flake-config = false' + export QUIXOS_PROTOCOL_EVALUATION_STORE="$TMPDIR/evaluation-store" + mkdir -p "$HOME" "$NIX_STATE_DIR" "$NIX_LOG_DIR" "$NIX_CONF_DIR" "$QUIXOS_PROTOCOL_EVALUATION_STORE/nix/store" + while IFS= read -r dependency; do + cp -a --no-preserve=ownership "$dependency" "$QUIXOS_PROTOCOL_EVALUATION_STORE$dependency" + done < ${closure}/store-paths + nix-store --store "local?root=$QUIXOS_PROTOCOL_EVALUATION_STORE" --load-db < ${closure}/registration + timeout --kill-after=5s 180s node ${./source-qualification.mjs} ${inputs} "$out" + ''