Inspect constructor values and contexts without unrelated derivation closures (#931)

source-qualification-metadata.nix

@@ -0,0 +1,34 @@
+# Read actual constructor attributes as comparison data. Their values AND
+# dependency contexts remain visible; none is substituted for a build input.
+derivation:
+let
+  fields = [
+    "name"
+    "system"
+    "src"
+    "stdenv"
+    "nativeBuildInputs"
+    "buildInputs"
+    "propagatedBuildInputs"
+    "propagatedNativeBuildInputs"
+    "configurePhase"
+    "buildPhase"
+    "checkPhase"
+    "installPhase"
+    "doCheck"
+    "strictDeps"
+    "outputs"
+    "__structuredAttrs"
+  ];
+  selected = builtins.intersectAttrs (builtins.listToAttrs (
+    map (name: {
+      inherit name;
+      value = null;
+    }) fields
+  )) derivation;
+in
+builtins.mapAttrs (_: value: {
+  type = builtins.typeOf value;
+  value = builtins.unsafeDiscardStringContext (toString value);
+  context = builtins.getContext (toString value);
+}) selected

source-qualification.mjs

@@ -34,6 +34,7 @@ function identity() {
     derivation = compiler.drvPath;
     yarn = compiler.yarn-freestanding.drvPath;
     source = toString compiler.src;
+    metadata = (import ((/. + ${nixString(source)}) + "/source-qualification-metadata.nix")) compiler;
   }`;
   const result = spawnSync(
     "nix",
@@ -52,17 +53,6 @@ function identity() {
   assert.equal(result.status, 0, result.error?.message ?? result.stderr);
   return JSON.parse(result.stdout);
 }
-function derivationData(file) {
-  const result = spawnSync(
-    "nix",
-    ["--store", "local?root=" + process.env.QUIXOS_PROTOCOL_EVALUATION_STORE, "derivation", "show", file],
-    { encoding: "utf8", timeout: 30000, maxBuffer: 2097152 },
-  );
-  assert.equal(result.status, 0, result.error?.message ?? result.stderr);
-  const entries = Object.values(JSON.parse(result.stdout));
-  assert.equal(entries.length, 1);
-  return entries[0];
-}
 function summarized(value) {
   const text = JSON.stringify(value) ?? "undefined";
   return text.length < 500
@@ -74,7 +64,7 @@ function summarized(value) {
       };
 }
 function structuralDifferences(actual, expected, field = "", result = []) {
-  if (field === "outputs" || field === "env.out" || JSON.stringify(actual) === JSON.stringify(expected)) return result;
+  if (JSON.stringify(actual) === JSON.stringify(expected)) return result;
   if (
     Array.isArray(actual) &&
     Array.isArray(expected) &&
@@ -99,15 +89,24 @@ function structuralDifferences(actual, expected, field = "", result = []) {
     for (const key of new Set([...Object.keys(actual), ...Object.keys(expected)]))
       structuralDifferences(actual[key], expected[key], field ? field + "." + key : key, result);
   } else {
-    result.push({ field, actual: summarized(actual), expected: summarized(expected) });
+    const difference = { field, actual: summarized(actual), expected: summarized(expected) };
+    if (typeof actual === "string" && typeof expected === "string" && Math.max(actual.length, expected.length) >= 500) {
+      let offset = 0;
+      while (offset < actual.length && offset < expected.length && actual[offset] === expected[offset]) offset++;
+      difference.offset = offset;
+      difference.actualExcerpt = actual.slice(Math.max(0, offset - 80), offset + 200);
+      difference.expectedExcerpt = expected.slice(Math.max(0, offset - 80), offset + 200);
+    }
+    result.push(difference);
   }
   return result;
 }
 function baselineDifference(baseline) {
+  const { metadata, ...identities } = baseline;
   console.log(
     "PROTOCOL_SOURCE_BASELINE: " +
       JSON.stringify({
-        actual: baseline,
+        actual: identities,
         expected: config.expected,
         expectedSource: config.expectedSource,
         evaluatorVersion: config.evaluatorVersion,
@@ -115,15 +114,10 @@ function baselineDifference(baseline) {
         narHashContext: config.narHashContext,
       }),
   );
-  try {
-    const actual = derivationData(baseline.derivation);
-    const expected = derivationData(config.productionDerivation);
-    console.log(
-      "PROTOCOL_SOURCE_BASELINE_DIFF: " + JSON.stringify(structuralDifferences(actual, expected)).slice(0, 10000),
-    );
-  } catch (error) {
-    console.log("PROTOCOL_SOURCE_BASELINE_DIAGNOSTIC_UNAVAILABLE: " + String(error.message).slice(0, 2000));
-  }
+  console.log(
+    "PROTOCOL_SOURCE_BASELINE_DIFF: " +
+      JSON.stringify(structuralDifferences(metadata, config.expectedMetadata)).slice(0, 10000),
+  );
 }
 async function edit(file, mutate, check) {
   const before = await fs.readFile(file, "utf8");
@@ -244,7 +238,9 @@ try {
     JSON.stringify(
       {
         codec: "quixos-protocol-source-qualification/1",
-        ...baseline,
+        derivation: baseline.derivation,
+        yarn: baseline.yarn,
+        source: baseline.source,
         cases,
         metadataContext: config.narHashContext,
         evaluatorVersion: config.evaluatorVersion,

source-qualification.nix

@@ -5,8 +5,7 @@
 }:
 let
   comparisonIdentity = import ../tools/local-first/derivation-identity.nix;
-  # Inspect the actual .drv as data, without its derivation-deep output context.
-  productionDerivation = builtins.unsafeDiscardOutputDependency library.drvPath;
+  inspect = import ./source-qualification-metadata.nix;
   source = pkgs.lib.fileset.toSource {
     root = ./.;
     fileset = pkgs.lib.fileset.unions [
@@ -23,6 +22,7 @@ let
       ./tsconfig.build.json
       ./tsconfig.test.json
       ./default.nix
+      ./source-qualification-metadata.nix
       ./yarn-project.nix
       ./flake.nix
       ./README.md
@@ -30,23 +30,19 @@ let
   };
   inputs = pkgs.writeText "protocol-source-inputs.json" (
     builtins.toJSON {
-      inherit source nixpkgsNarHash productionDerivation;
+      inherit source nixpkgsNarHash;
       narHashContext = builtins.getContext nixpkgsNarHash;
       evaluatorVersion = builtins.nixVersion;
       pinnedNixVersion = pkgs.nix.version;
       nixpkgs = pkgs.path;
       system = pkgs.stdenv.hostPlatform.system;
       expected = comparisonIdentity library;
+      expectedMetadata = inspect library;
       expectedSource = builtins.unsafeDiscardStringContext (toString library.src);
       yarn = comparisonIdentity library.yarn-freestanding;
     }
   );
-  closure = pkgs.closureInfo {
-    rootPaths = [
-      pkgs.path
-      productionDerivation
-    ];
-  };
+  closure = pkgs.closureInfo { rootPaths = [ pkgs.path ]; };
 in
 pkgs.runCommand "quixos-protocol-source-check"
   {