Preserve compiler metadata context and diagnose exact identity differences (#931)

source-qualification.mjs

@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
 
 const config = JSON.parse(await fs.readFile(process.argv[2], "utf8"));
 const output = process.argv[3];
@@ -23,10 +24,11 @@ async function writable(directory) {
 }
 function identity() {
   const expression = `let
-    pkgs = import ${nixString(config.nixpkgs)} { system = ${nixString(config.system)}; };
+    pkgs = import (/. + ${nixString(config.nixpkgs)}) { system = ${nixString(config.system)}; };
-    compiler = import (${nixString(source)} + "/default.nix") {
+    compiler = import ((/. + ${nixString(source)}) + "/default.nix") {
       inherit pkgs;
-      nixpkgsNarHash = ${nixString(config.nixpkgsNarHash)};
+      nixpkgsNarHash = builtins.appendContext ${nixString(config.nixpkgsNarHash)}
+        (builtins.fromJSON ${nixString(JSON.stringify(config.narHashContext))});
     };
   in {
     derivation = compiler.drvPath;
@@ -50,6 +52,79 @@ function identity() {
   assert.equal(result.status, 0, result.error?.message ?? result.stderr);
   return JSON.parse(result.stdout);
 }
+function derivationData(file) {
+  const result = spawnSync(
+    "nix",
+    ["--store", "local?root=" + process.env.QUIXOS_PROTOCOL_EVALUATION_STORE, "derivation", "show", file],
+    { encoding: "utf8", timeout: 30000, maxBuffer: 2097152 },
+  );
+  assert.equal(result.status, 0, result.error?.message ?? result.stderr);
+  const entries = Object.values(JSON.parse(result.stdout));
+  assert.equal(entries.length, 1);
+  return entries[0];
+}
+function summarized(value) {
+  const text = JSON.stringify(value) ?? "undefined";
+  return text.length < 500
+    ? value
+    : {
+        bytes: Buffer.byteLength(text),
+        sha256: createHash("sha256").update(text).digest("hex"),
+        prefix: text.slice(0, 180),
+      };
+}
+function structuralDifferences(actual, expected, field = "", result = []) {
+  if (field === "outputs" || field === "env.out" || JSON.stringify(actual) === JSON.stringify(expected)) return result;
+  if (
+    Array.isArray(actual) &&
+    Array.isArray(expected) &&
+    [...actual, ...expected].every((x) => typeof x === "string")
+  ) {
+    result.push({
+      field,
+      onlyActual: actual.filter((x) => !expected.includes(x)),
+      onlyExpected: expected.filter((x) => !actual.includes(x)),
+      actualCount: actual.length,
+      expectedCount: expected.length,
+      sequenceChanged: true,
+    });
+  } else if (
+    actual &&
+    expected &&
+    typeof actual === "object" &&
+    typeof expected === "object" &&
+    !Array.isArray(actual) &&
+    !Array.isArray(expected)
+  ) {
+    for (const key of new Set([...Object.keys(actual), ...Object.keys(expected)]))
+      structuralDifferences(actual[key], expected[key], field ? field + "." + key : key, result);
+  } else {
+    result.push({ field, actual: summarized(actual), expected: summarized(expected) });
+  }
+  return result;
+}
+function baselineDifference(baseline) {
+  console.log(
+    "PROTOCOL_SOURCE_BASELINE: " +
+      JSON.stringify({
+        actual: baseline,
+        expected: config.expected,
+        expectedSource: config.expectedSource,
+        evaluatorVersion: config.evaluatorVersion,
+        pinnedNixVersion: config.pinnedNixVersion,
+        narHashContext: config.narHashContext,
+      }),
+  );
+  try {
+    const actual = derivationData(baseline.derivation);
+    const expected = derivationData(config.productionDerivation);
+    console.log(
+      "PROTOCOL_SOURCE_BASELINE_DIFF: " + JSON.stringify(structuralDifferences(actual, expected)).slice(0, 10000),
+    );
+  } catch (error) {
+    console.log("PROTOCOL_SOURCE_BASELINE_DIAGNOSTIC_UNAVAILABLE: " + String(error.message).slice(0, 2000));
+  }
+}
 async function edit(file, mutate, check) {
   const before = await fs.readFile(file, "utf8");
   try {
@@ -69,6 +144,7 @@ try {
   await fs.cp(config.source, source, { recursive: true });
   await writable(source);
   const baseline = identity();
+  if (baseline.derivation !== config.expected) baselineDifference(baseline);
   await record("real production compiler and generated Yarn launcher identities", async () => {
     assert.equal(baseline.derivation, config.expected);
     assert.equal(baseline.yarn, config.yarn);
@@ -170,6 +246,9 @@ try {
         codec: "quixos-protocol-source-qualification/1",
         ...baseline,
         cases,
+        metadataContext: config.narHashContext,
+        evaluatorVersion: config.evaluatorVersion,
+        pinnedNixVersion: config.pinnedNixVersion,
         nestedBuilds: false,
         guestExecution: false,
       },

source-qualification.nix

@@ -5,6 +5,8 @@
 }:
 let
   comparisonIdentity = import ../tools/local-first/derivation-identity.nix;
+  # Inspect the actual .drv as data, without its derivation-deep output context.
+  productionDerivation = builtins.unsafeDiscardOutputDependency library.drvPath;
   source = pkgs.lib.fileset.toSource {
     root = ./.;
     fileset = pkgs.lib.fileset.unions [
@@ -28,14 +30,23 @@ let
   };
   inputs = pkgs.writeText "protocol-source-inputs.json" (
     builtins.toJSON {
-      inherit source nixpkgsNarHash;
+      inherit source nixpkgsNarHash productionDerivation;
+      narHashContext = builtins.getContext nixpkgsNarHash;
+      evaluatorVersion = builtins.nixVersion;
+      pinnedNixVersion = pkgs.nix.version;
       nixpkgs = pkgs.path;
       system = pkgs.stdenv.hostPlatform.system;
       expected = comparisonIdentity library;
+      expectedSource = builtins.unsafeDiscardStringContext (toString library.src);
       yarn = comparisonIdentity library.yarn-freestanding;
     }
   );
-  closure = pkgs.closureInfo { rootPaths = [ pkgs.path ]; };
+  closure = pkgs.closureInfo {
+    rootPaths = [
+      pkgs.path
+      productionDerivation
+    ];
+  };
 in
 pkgs.runCommand "quixos-protocol-source-check"
   {