Publish quixos-nix-helpers from e4393946ba63cfef75580f6aa6ba840035e7d43c

Show per-export placement, static eligibility blockers, bounded metadata and live pool/idle/timing diagnostics. Attach inspector updates through one private shard stream and the existing authenticated SSE hub without waking graph views or VMs. Fix packed agent preparation to use OpenSSH and the worker’s own private systemd credential.

.quixos-subtree-source.json

@@ -1,7 +1,7 @@
 {
   "version": 1,
-  "sourceRepo": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos",
-  "sourceCommit": "932fe7714761fb8b5db4bd6401357cdff762a428",
+  "sourceRepo": "https://gitea.internal.quixos.org/quixos/quixos.git",
+  "sourceCommit": "e4393946ba63cfef75580f6aa6ba840035e7d43c",
   "sourcePath": "quixos-instance/quixos-nix-helpers",
   "exportName": "quixos-nix-helpers",
   "mirrorRemote": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git"

PORTABLE_ARTIFACTS.md

@@ -1,5 +1,11 @@
 # Checked capability and component artifacts
 
+This is the **current helper/output format**, not the core package abstraction.
+See [the architecture](../../docs/ARCHITECTURE.md) and
+[current gaps](../../docs/ARCHITECTURE_STATUS.md). In particular, the universal
+artifact inventory and component-only omission of a server are implementation
+gaps to resolve, not requirements for arbitrary package servers.
+
 `mkCaminoArtifacts` is the fresh package builder. It regenerates candidate
 bindings, typechecks the entry's dependency closure, bundles declared targets,
 generates a process service when requested, and emits content manifests and
@@ -55,6 +61,25 @@ with their own versions. Authored source symlinks and non-bundled portable impor
 other than the SDK and server Node builtins are rejected. This is build closure
 checking, not a purity proof or security sandbox.
 
+`mkCaminoYarnDependencies` installs the locked dependency recipe and exports its
+installed Node dependency tree, including native package results and executable
+links. Generated Yarn install/linker state stays outside runtime outputs. Yarn
+describes its install-state file as an optimization cache in its
+[configuration reference](https://yarnpkg.com/configuration/yarnrc#installStatePath).
+This helper output is a dependency directory for a build, not a standalone
+Package or a runnable Yarn project. Its `devShell` supplies freestanding Yarn for
+editable projects; do not run Yarn against the readonly dependency output.
+
+Use the shared `installDependencies` assembly for consumers: authored libraries
+are copied into the consumer's `node_modules`, where Node can find their
+transitive dependencies, while selected SDK links retain their canonical type
+identity. Reserved SDK substitutions fail checks. Web Studio uses this same
+assembly rather than retaining a full Yarn project just to resolve UI libraries.
+Its dependency inputs contain manifests, lock and Yarn configuration/plugin
+recipe; ordinary Studio source edits do not enter that dependency derivation.
+Recipes with local/workspace dependencies must still include their needed source.
+This is language-specific build machinery; it adds no core Package file contract.
+
 The core/Automerge bundle and Replicache bundle are built once. Browser modules
 import the core through a content-addressed gateway URL, independent of package
 identity. Each manifest covers the shared bytes it uses. The gateway retains its
@@ -105,7 +130,7 @@ and supplies an explicit SDK for value codecs, identities and declared graph por
 service = {
   restricted = true;
   limits = {
-    deadlineMs = 4000;
+    deadlineMs = 10000;
     maxBytes = 1048576;
     maxPortCalls = 1000;
     maxQueued = 8;
@@ -122,6 +147,14 @@ invocation identity; authored code cannot choose another frame or workspace.
 Native graph grants and transaction commit remain enforced by Camino outside
 the guest. Initialization and synchronous loops are subject to the same deadline.
 
+The default root/Package Invocation wall deadline is 10 seconds. The separate
+root CPU allowance remains at most 4 CPU seconds; extra wall time accommodates
+queueing and graph-port waits. Ordinary graph database transactions and idle
+transactions allow 30 seconds, and ordinary client HTTP requests allow 45 seconds
+so server failures can be returned. Nested calls inherit the remaining root
+budget; explicit smaller limits still apply. Restore/activation jobs retain their
+separate budgets. These defaults are starting limits, not latency targets.
+
 Direct network access, native filesystem imports, ambient clock and entropy,
 parent environment credentials, and unlisted modules are denied. Explicit dates,
 `Intl.Segmenter`, deterministic hashes and normal JavaScript computation remain

restricted-execution/default.nix

@@ -20,24 +20,16 @@ let
     runtime = nativeProbe;
     executable = "${nativeProbe}/bin/probe";
   };
-  identityLauncher =
-    pkgs.runCommand "quixos-restricted-identity-launcher"
-      {
-        nativeBuildInputs = [ pkgs.stdenv.cc ];
-      }
-      ''
-        mkdir -p "$out/bin"
-        $CC -O2 -Wall -Wextra -Werror ${./identity-launcher.c} \
-          -DSANDBOX_EXECUTABLE='"${sandbox}/bin/quixos-restricted-sandbox"' \
-          -o "$out/bin/quixos-restricted-engine"
-      '';
+  mkIdentityLauncher =
+    settings: import ./identity-launcher.nix ({ inherit pkgs sandbox; } // settings);
+  identityLauncher = mkIdentityLauncher { };
   # Transaction-port/value adapters do not need the browser's CRDT/Wasm engine.
   # Keep this surface explicit; it is bundled into checked restricted handlers.
   sdk = pkgs.runCommand "quixos-restricted-sdk" { nativeBuildInputs = [ pkgs.esbuild ]; } ''
     mkdir -p "$out"
     cat > entry.mjs <<'JS'
     export {definePortableRegistry,QxReject} from '${core}/dist/portable.js';
-    export {encodeArgument,decodeArgument,canonicalJson,parseJson} from '${core}/dist/value.js';
+    export {encodeArgument,decodeArgument,canonicalJson,parseJson,hashJson} from '${core}/dist/value.js';
     export {workspaceId,operationId,objectId,entryId,hashIdentity} from '${core}/dist/identity.js';
     export {reference} from '${core}/dist/references.js';
     export {CodecError,ExecutionError} from '${core}/dist/limits.js';
@@ -73,6 +65,7 @@ in
     workerd
     sandbox
     identityLauncher
+    mkIdentityLauncher
     nativeProbeSandbox
     sdk
     runtime

restricted-execution/identity-launcher.c

@@ -20,17 +20,17 @@ static void fail(const char *message) {
 }
 
 int main(int argc, char **argv) {
-  const char *root = "/run/quixos-restricted-engines/";
+  const char *root = ENGINE_DIRECTORY "/";
   uid_t caller = getuid();
   gid_t group = getgid();
-  struct passwd *owner = getpwnam("quixos-graph");
+  struct passwd *owner = getpwnam(AUTHORITY_USER);
   if (geteuid() != 0) fail("privileged wrapper unavailable");
   if (!owner) fail("authority identity unavailable");
   if (caller != owner->pw_uid) fail("caller identity mismatch");
   if (argc != 3) fail("invalid arguments");
   char *end;
   unsigned long number = strtoul(argv[2], &end, 10);
-  if (!*argv[2] || *end || number < 610000 || number >= 610064)
+  if (!*argv[2] || *end || number < ENGINE_FIRST_UID || number >= ENGINE_FIRST_UID + ENGINE_IDENTITY_COUNT)
     fail("invalid engine identity");
   uid_t identity = (uid_t)number;
   if (strlen(argv[1]) != strlen(root) + 16 || strncmp(argv[1], root, strlen(root)))
@@ -39,7 +39,7 @@ int main(int argc, char **argv) {
   if (strncmp(name, "qx-engine-", 10) ||
       strlen(name) != 16 || strspn(name + 10, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") != 6)
     fail("invalid engine directory");
-  int parent = open("/run/quixos-restricted-engines", O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
+  int parent = open(ENGINE_DIRECTORY, O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
   struct stat info;
   if (parent < 0 || fstat(parent, &info) || info.st_uid != caller || (info.st_mode & 066))
     fail("invalid engine root");

restricted-execution/identity-launcher.nix

@@ -0,0 +1,28 @@
+# Trusted host construction only. Authored Workers cannot choose the caller,
+# reserved identities, engine directory or sandbox executable.
+{
+  pkgs,
+  sandbox,
+  authorityUser ? "quixos-graph",
+  engineDirectory ? "/run/quixos-restricted-engines",
+  firstUid ? 610000,
+  identityCount ? 64,
+}:
+assert builtins.match "[a-z][a-z0-9_-]{0,63}" authorityUser != null;
+assert builtins.match "/run/[a-zA-Z0-9_/-]+" engineDirectory != null;
+assert !(pkgs.lib.hasInfix ".." engineDirectory);
+assert !(pkgs.lib.hasSuffix "/" engineDirectory);
+assert builtins.stringLength engineDirectory < 80;
+assert builtins.isInt firstUid && builtins.isInt identityCount;
+assert firstUid >= 610000 && firstUid + identityCount <= 2147483647;
+assert identityCount >= 1 && identityCount <= 64;
+pkgs.runCommand "quixos-restricted-identity-launcher" { nativeBuildInputs = [ pkgs.stdenv.cc ]; } ''
+  mkdir -p "$out/bin"
+  $CC -O2 -Wall -Wextra -Werror ${./identity-launcher.c} \
+    -DSANDBOX_EXECUTABLE='"${sandbox}/bin/quixos-restricted-sandbox"' \
+    -DAUTHORITY_USER='"${authorityUser}"' \
+    -DENGINE_DIRECTORY='"${engineDirectory}"' \
+    -DENGINE_FIRST_UID=${toString firstUid} \
+    -DENGINE_IDENTITY_COUNT=${toString identityCount} \
+    -o "$out/bin/quixos-restricted-engine"
+''

restricted-execution/identity-module.nix

@@ -2,23 +2,45 @@
   executor,
   enable,
   controlGroup,
+  authorityUser ? "quixos-graph",
+  engineDirectory ? "/run/quixos-restricted-engines",
+  firstUid ? 610000,
+  identityCount ? 64,
+  wrapperName ? "quixos-restricted-engine",
 }:
 { config, lib, ... }:
 {
   config = lib.mkIf enable {
     assertions = [
       {
-        assertion = lib.all (user: user.uid == null || user.uid < 610000 || user.uid >= 610064) (
-          builtins.attrValues config.users.users
-        );
-        message = "UIDs 610000–610063 are reserved for restricted engine identities.";
+        assertion = lib.all (
+          user: user.uid == null || user.uid < firstUid || user.uid >= firstUid + identityCount
+        ) (builtins.attrValues config.users.users);
+        message = "UIDs ${toString firstUid}–${
+          toString (firstUid + identityCount - 1)
+        } are reserved for restricted engine identities.";
+      }
+      {
+        assertion = lib.all (
+          group: group.gid == null || group.gid < firstUid || group.gid >= firstUid + identityCount
+        ) (builtins.attrValues config.users.groups);
+        message = "Restricted engine GIDs must not overlap host groups.";
       }
     ];
     systemd.tmpfiles.rules = [
-      "d /run/quixos-restricted-engines 0711 quixos-graph ${controlGroup} -"
+      "d ${engineDirectory} 0711 ${authorityUser} ${controlGroup} -"
     ];
-    security.wrappers.quixos-restricted-engine = {
-      source = "${executor.identityLauncher}/bin/quixos-restricted-engine";
+    security.wrappers.${wrapperName} = {
+      source = "${
+        executor.mkIdentityLauncher {
+          inherit
+            authorityUser
+            engineDirectory
+            firstUid
+            identityCount
+            ;
+        }
+      }/bin/quixos-restricted-engine";
       owner = "root";
       group = controlGroup;
       permissions = "u+rx,g+rx,o-rwx";

restricted-execution/service.mjs

@@ -7,6 +7,32 @@ import { tmpdir } from "node:os";
 import { join, dirname } from "node:path";
 
 const abi = "quixos-transaction-ports-v1";
+/** Reserved supervisor faults are not authored domain rejections. A graph
+ * transaction must roll back without recording a permanent rejected outcome
+ * when its executor dies, exceeds its limits or cannot start. */
+export const restrictedExecutionFailureCodes = Object.freeze([
+  "RESTRICTED_ABI_MISMATCH",
+  "RESTRICTED_BACKPRESSURE",
+  "RESTRICTED_CPU_ACCOUNTING",
+  "RESTRICTED_DEADLINE",
+  "RESTRICTED_DIAGNOSTIC_LIMIT",
+  "RESTRICTED_DUPLICATE_INVOCATION",
+  "RESTRICTED_ENGINE_EXITED",
+  "RESTRICTED_ENGINE_LIMIT",
+  "RESTRICTED_ENTRY_LIMIT",
+  "RESTRICTED_IDENTITY_UNAVAILABLE",
+  "RESTRICTED_MESSAGE_LIMIT",
+  "RESTRICTED_PACKAGE_FAILED",
+  "RESTRICTED_PORT_FAILED",
+  "RESTRICTED_PORT_LIMIT",
+  "RESTRICTED_PROCESS_LIMIT",
+  "RESTRICTED_QUEUE_FULL",
+  "RESTRICTED_SDK_LIMIT",
+  "RESTRICTED_SOURCE_LIMIT",
+  "RESTRICTED_START_FAILED",
+  "RESTRICTED_UNKNOWN_INVOCATION",
+  "RESTRICTED_UNKNOWN_PORT",
+]);
 const fixedError = (code) => Object.assign(Error(code), { code });
 const object = (v) => !!v && typeof v === "object" && !Array.isArray(v);
 const absolute = (value) =>
@@ -108,7 +134,7 @@ export async function serveRestrictedRegistry(config, io = process) {
   const limits = {
     maxBytes: config.maxBytes ?? 1024 * 1024,
     maxPortCalls: config.maxPortCalls ?? 1000,
-    deadlineMs: config.deadlineMs ?? 4000,
+    deadlineMs: config.deadlineMs ?? 10000,
     maxQueued: config.maxQueued ?? 8,
     maxActive: config.maxActive ?? 2,
     maxNested: config.maxNested ?? 4,
@@ -148,6 +174,11 @@ export async function serveRestrictedRegistry(config, io = process) {
     engines = new Set(),
     idleEngines = [];
   const metrics = { starts: 0, reused: 0, completed: 0, peakRSSBytes: 0 };
+  const changed = () => {
+    try {
+      config.onChange?.();
+    } catch {}
+  };
   let buffered = Buffer.alloc(0),
     stopped = false,
     failure;
@@ -234,16 +265,19 @@ export async function serveRestrictedRegistry(config, io = process) {
       }
       const engine = call.engine;
       if (engine) {
-        // Revoke the frame token before returning an engine to the idle pool.
-        // A delayed child request can never acquire the next call's authority.
-        if (engine.current === call) engine.current = null;
         if (!stopped && reason.code === "INVOCATION_ENDED" && !usageFailure && !engine.destroying) {
           const rss = (await engineUsage(engine)).memoryBytes;
           metrics.peakRSSBytes = Math.max(metrics.peakRSSBytes, rss);
           // Bound retained engine heaps as well as lifetime. Hard aggregate
           // physical memory remains the supervising process/cgroup's contract.
-          if (rss <= 128 * 1024 * 1024) {
+          // Keep the lease busy through every await. Another artifact can evict
+          // an idle lease immediately; never publish an engine still retiring
+          // or one whose process died while its final usage read was pending.
+          if (rss <= 128 * 1024 * 1024 && !stopped && !engine.destroying && engine.current === call) {
+            // Revoke the frame token atomically with publishing idle capacity.
+            engine.current = null;
             idleEngines.push(engine);
+            engine.idleUntil = Date.now() + limits.engineIdleMs;
             engine.idleTimer = setTimeout(() => void destroyEngine(engine).catch(fail), limits.engineIdleMs);
           } else await destroyEngine(engine);
         } else await destroyEngine(engine);
@@ -300,6 +334,7 @@ export async function serveRestrictedRegistry(config, io = process) {
       } finally {
         engines.delete(engine);
         await engine.release?.();
+        changed();
       }
     })();
     return engine.destroying;
@@ -418,6 +453,7 @@ export async function serveRestrictedRegistry(config, io = process) {
       },
     );
     metrics.starts++;
+    changed();
     engine.exited = new Promise((resolve) => {
       engine.child.once("exit", resolve);
       engine.child.once("error", resolve);
@@ -462,6 +498,9 @@ export async function serveRestrictedRegistry(config, io = process) {
         executionMs: call.executionMs ?? null,
         totalMs: performance.now() - call.receivedAt,
       };
+      metrics.lastFailure =
+        reply.status === "failed" && /^[A-Z0-9_]{1,128}$/.test(reply.error?.code) ? reply.error.code : null;
+      changed();
       send(call.cancellationReply ?? reply);
       pump();
     })());
@@ -470,6 +509,7 @@ export async function serveRestrictedRegistry(config, io = process) {
     const { message } = call;
     call.startedAt = performance.now();
     active.set(message.invocation, call);
+    changed();
     const terminate = (code) => {
       if (!call.open) return;
       void finish(call, fixedError(code), {
@@ -687,6 +727,9 @@ export async function serveRestrictedRegistry(config, io = process) {
       queued: queued.length,
       engines: engines.size,
       idle: idleEngines.length,
+      idleDeadline: idleEngines.length
+        ? new Date(Math.min(...idleEngines.map((engine) => engine.idleUntil))).toISOString()
+        : null,
     }),
   };
 }

restricted-execution/service.test.mjs

@@ -307,6 +307,47 @@ test("the checked portable SDK keeps codecs and nested graph ports inside the fr
     await f.close();
   }
 });
+test("checked handlers hash canonical values without ambient capabilities", async () => {
+  const sdk = process.env.QUIXOS_TEST_SDK;
+  const C = await import(sdk);
+  const workspace = C.workspaceId("workspace:restricted");
+  const text = { kind: "scalar", name: "string" };
+  const value = {
+    kind: "record",
+    fields: {
+      text,
+      options: {
+        kind: "record",
+        fields: {
+          level: { kind: "scalar", name: "int32" },
+          enabled: { kind: "scalar", name: "bool" },
+        },
+      },
+    },
+  };
+  const declarations = [
+    { id: "test", name: "test", effect: "snapshot", receiver: null, input: value, output: text, methods: [] },
+  ];
+  const source = `import {definePortableRegistry,hashJson} from 'qx:core'; export const registry=definePortableRegistry(${JSON.stringify(declarations)},{test:ctx=>hashJson('test-content',ctx.input)});`;
+  const f = await fixture(source, { sdk: [{ name: "qx:core", path: sdk }] });
+  try {
+    const input = { text: "雨后的书店", options: { level: 3, enabled: true } };
+    let expected;
+    for (const candidate of [input, { options: input.options, text: input.text }, { ...input, text: "別の文" }]) {
+      f.start(C.encodeArgument(value, candidate, { workspace }));
+      const result = await f.receive();
+      assert.equal(result.status, "returned");
+      const actual = C.decodeArgument(text, result.output, { workspace });
+      assert.equal(actual, await C.hashJson("test-content", candidate));
+      if (candidate.text === input.text) {
+        expected ??= actual;
+        assert.equal(actual, expected, "Object key order must not invalidate content caches");
+      } else assert.notEqual(actual, expected, "Content changes must invalidate the key");
+    }
+  } finally {
+    await f.close();
+  }
+});
 test("ambient clock, entropy, network and native imports are denied", async () => {
   for (const code of [
     "Date.now()",
@@ -529,12 +570,25 @@ test("an expired frame token cannot borrow a later invocation's broker", async (
 });
 
 test("idle engines evict without a graph poll and restart with fresh state", async () => {
-  const f = await fixture(registry("return request.input;"), { engineIdleMs: 30 });
+  const updates = [];
+  const f = await fixture(registry("return request.input;"), {
+    engineIdleMs: 30,
+    onChange: () => updates.push(f.stats()),
+  });
   try {
     f.start("first");
     assert.equal((await f.receive()).output, "first");
     await new Promise((resolve) => setTimeout(resolve, 100));
     assert.equal(f.stats().engines, 0);
+    assert.ok(
+      updates.some((s) => s.active > 0),
+      "Inspectors observe active invocation admission",
+    );
+    assert.ok(
+      updates.some((s) => s.idle > 0 && s.idleDeadline),
+      "Idle eviction deadline is visible",
+    );
+    assert.equal(updates.at(-1).engines, 0, "Eviction pushes an update without a graph poll");
     f.start("second");
     assert.equal((await f.receive()).output, "second");
     assert.equal(f.stats().starts, 2);
@@ -632,3 +686,39 @@ test("the shared engine budget evicts idle code families and rejects busy oversu
   }
   assert.equal(engineBudget.stats().engines, 0);
 });
+
+test("engine retirement accounting keeps its admission lease busy", async () => {
+  const observations = [];
+  let idle;
+  const engineBudget = new RestrictedEngineBudget(1, async () => ({
+    usage: async () => {
+      observations.push(idle());
+      await new Promise((done) => setImmediate(done));
+      return { cpuMicroseconds: 0, memoryBytes: 1024 };
+    },
+    release: async () => {},
+  }));
+  const acquire = engineBudget.acquire.bind(engineBudget);
+  engineBudget.acquire = (isIdle, evict) => {
+    idle = isIdle;
+    return acquire(isIdle, evict);
+  };
+  const f = await fixture(registry("return request.input;"), { engineBudget });
+  try {
+    f.start("first");
+    assert.equal((await f.receive()).output, "first");
+    assert.ok(observations.length > 0);
+    assert.ok(
+      observations.every((value) => !value),
+      "An engine must not become evictable during asynchronous retirement accounting",
+    );
+    assert.equal(f.stats().idle, 1);
+    assert.equal(idle(), true);
+    f.start("second");
+    assert.equal((await f.receive()).output, "second");
+    assert.equal(f.stats().starts, 1);
+  } finally {
+    await f.close();
+  }
+  assert.equal(engineBudget.stats().engines, 0);
+});

yarn-dependencies.nix

@@ -13,6 +13,9 @@ let
         test -d node_modules
         mkdir -p "$out"
         cp -a node_modules/. "$out/"
+        # Node resolves installed packages directly. This generated index is for
+        # Yarn's linker, not a runtime input; do not export build-session state.
+        rm -f "$out/.yarn-state.yml"
         runHook postInstall
       '';
     };