Confine restricted engines before launch and preserve the reader recovery baseline

PORTABLE_ARTIFACTS.md

@@ -113,8 +113,9 @@ service = {
 };
 ```
 
-Each invocation gets a fresh engine process. The adapter admits one active frame
+Each invocation gets a fresh Worker in a reusable engine process. An engine runs
-and a bounded FIFO; queue wait consumes the deadline. Overflow is an explicit
+one active frame; the adapter has bounded concurrency and queueing. Queue wait
+consumes the deadline. Overflow is an explicit
 `RESTRICTED_QUEUE_FULL` result. Cancellation kills the engine process group and
 removes its sockets before acknowledgement. All graph calls inherit the trusted
 invocation identity; authored code cannot choose another frame or workspace.
@@ -127,11 +128,15 @@ parent environment credentials, and unlisted modules are denied. Explicit dates,
 available. Host-managed HTTP bridges expose only the current frame's declared
 graph ports. Authored serialization is untrusted protocol input.
 
-This is an inner execution restriction, **not a hardened shared-process sandbox**.
+The packaged runtime uses the platform's Bubblewrap/seccomp launcher. Only the
-One workspace owns the surrounding VM. Systemd limits the entire package cgroup,
+engine's read-only runtime closure and private broker directory are mounted;
-including adapter and child engines; memory exhaustion terminates that group.
+native filesystem/process/network escape probes qualify that boundary. Systemd
-No per-isolate memory protection is claimed. A guest user can control its VM,
+limits the whole package in a workspace VM. Shared authority additionally charges
-so authoritative graph grants and credentials must remain outside it.
+each engine to its own cgroup before launch. No per-isolate memory protection or
+Cloudflare hosted side-channel defenses are claimed. See the
+[runtime decision](../../docs/RESTRICTED_RUNTIME_DECISION.md#os-process-confinement-implementation)
+for the exact boundary and residual risks. A guest user can control its VM, so
+authoritative graph grants and credentials must remain outside it.
 
 Qualification commands:
 

restricted-execution/default.nix

@@ -1,6 +1,25 @@
 { pkgs, core }:
 let
   workerd = import ./workerd.nix { inherit pkgs; };
+  sandbox = import ./sandbox.nix {
+    inherit pkgs;
+    runtime = workerd;
+    executable = "${workerd}/bin/workerd";
+  };
+  nativeProbe =
+    pkgs.runCommand "quixos-restricted-native-probe"
+      {
+        nativeBuildInputs = [ pkgs.stdenv.cc ];
+      }
+      ''
+        mkdir -p "$out/bin"
+        $CC -O2 -Wall -Wextra -Werror ${./sandbox-probe.c} -o "$out/bin/probe"
+      '';
+  nativeProbeSandbox = import ./sandbox.nix {
+    inherit pkgs;
+    runtime = nativeProbe;
+    executable = "${nativeProbe}/bin/probe";
+  };
   # Transaction-port/value adapters do not need the browser's CRDT/Wasm engine.
   # Keep this surface explicit; it is bundled into checked restricted handlers.
   sdk = pkgs.runCommand "quixos-restricted-sdk" { nativeBuildInputs = [ pkgs.esbuild ]; } ''
@@ -23,6 +42,7 @@ let
         makeWrapper ${pkgs.nodejs_24}/bin/node "$out/bin/quixos-restricted-registry" \
           --add-flags "$out/lib/service.mjs" \
           --set QUIXOS_RESTRICTED_WORKERD ${workerd}/bin/workerd \
+          --set QUIXOS_RESTRICTED_SANDBOX ${sandbox}/bin/quixos-restricted-sandbox \
           --set QUIXOS_RESTRICTED_WORKER "$out/lib/worker.mjs"
       '';
   qualification =
@@ -40,6 +60,8 @@ in
 {
   inherit
     workerd
+    sandbox
+    nativeProbeSandbox
     sdk
     runtime
     qualification

restricted-execution/sandbox-probe.c

@@ -0,0 +1,64 @@
+#define _GNU_SOURCE
+#include <errno.h>
+#include <fcntl.h>
+#include <sched.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ptrace.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+static void check(int condition, const char *label) {
+  if (!condition) {
+    fprintf(stderr, "ESCAPE: %s (%s)\n", label, strerror(errno));
+    exit(1);
+  }
+}
+int main(void) {
+  for (int fd = 3; fd < 256; fd++)
+    check(fcntl(fd, F_GETFD) < 0 && errno == EBADF, "inherited descriptor");
+  check(!getenv("QUIXOS_VM_SECRET"), "parent environment");
+  check(open("/etc/test-secret", O_RDONLY) < 0, "authority filesystem");
+  check(open("/sys/fs/cgroup/cgroup.procs", O_WRONLY) < 0, "resource controls");
+  check(open("/proc/1/root/etc/test-secret", O_RDONLY) < 0, "proc root escape");
+  check(socket(AF_INET, SOCK_STREAM, 0) < 0 && errno == EPERM, "IPv4 socket");
+  check(socket(AF_INET6, SOCK_STREAM, 0) < 0 && errno == EPERM, "IPv6 socket");
+  check(socket(AF_NETLINK, SOCK_RAW, 0) < 0 && errno == EPERM, "netlink socket");
+  check(ptrace(PTRACE_TRACEME, 0, NULL, NULL) < 0 && errno == EPERM, "ptrace");
+  check(unshare(CLONE_NEWUSER) < 0, "nested user namespace");
+  pid_t child = fork();
+  if (child == 0) _exit(2);
+  check(child < 0 && errno == EPERM, "subprocess");
+  check(access("/bin/sh", X_OK) < 0, "ambient executable");
+  check(getppid() <= 1, "host process visibility");
+  struct sockaddr_un address = {.sun_family = AF_UNIX};
+  int outside = socket(AF_UNIX, SOCK_STREAM, 0);
+  check(outside >= 0, "Unix socket support");
+  strcpy(address.sun_path, "/run/quixos-native-authority/test.sock");
+  check(connect(outside, (struct sockaddr *)&address, sizeof(address)) < 0,
+    "authority Unix socket");
+  close(outside);
+  outside = socket(AF_UNIX, SOCK_STREAM, 0);
+  memset(address.sun_path, 0, sizeof(address.sun_path));
+  strcpy(address.sun_path + 1, "quixos-native-authority");
+  check(connect(outside, (struct sockaddr *)&address,
+    offsetof(struct sockaddr_un, sun_path) + 1 + strlen(address.sun_path + 1)) < 0,
+    "authority abstract socket");
+  close(outside);
+  char directory[4096];
+  check(getcwd(directory, sizeof(directory)) != NULL, "engine directory");
+  check(snprintf(address.sun_path, sizeof(address.sun_path), "%s/broker.sock", directory) <
+      (int)sizeof(address.sun_path), "broker path length");
+  int broker = socket(AF_UNIX, SOCK_STREAM, 0);
+  check(broker >= 0 && connect(broker, (struct sockaddr *)&address, sizeof(address)) == 0,
+      "authorized broker");
+  check(write(broker, "sandbox-probe", 13) == 13, "broker write");
+  close(broker);
+  puts("RESTRICTED_NATIVE_SANDBOX_PASSED");
+  return 0;
+}

restricted-execution/sandbox.c

@@ -0,0 +1,77 @@
+#define _GNU_SOURCE
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+/* Trusted, single-process admission gate. The supervisor charges this PID to
+ * its engine cgroup before releasing stdin. All subsequently created namespace
+ * helpers and threads inherit that charge. No authored code runs here. */
+static void fail(const char *message) {
+  fprintf(stderr, "RESTRICTED_SANDBOX: %s\n", message);
+  exit(125);
+}
+
+int main(int argc, char **argv) {
+  struct stat info;
+  if (argc != 2 || argv[1][0] != '/' || strstr(argv[1], "/../") ||
+      lstat(argv[1], &info) || !S_ISDIR(info.st_mode) ||
+      info.st_uid != getuid() || (info.st_mode & 077))
+    fail("invalid engine directory");
+  char gate;
+  if (read(STDIN_FILENO, &gate, 1) != 1 || gate != 1)
+    fail("engine admission was not released");
+  int input = open("/dev/null", O_RDONLY);
+  if (input < 0 || dup2(input, STDIN_FILENO) < 0)
+    fail("stdin");
+  close(input);
+  int filter = open(SANDBOX_FILTER, O_RDONLY);
+  if (filter < 0)
+    fail("seccomp filter");
+  char filter_fd[32];
+  snprintf(filter_fd, sizeof(filter_fd), "%d", filter);
+  char *args[4096];
+  size_t count = 0;
+#define ARG(value) args[count++] = (value)
+  ARG(SANDBOX_BWRAP);
+  ARG("--unshare-all");
+  ARG("--unshare-user");
+  ARG("--die-with-parent");
+  ARG("--disable-userns");
+  ARG("--assert-userns-disabled");
+  ARG("--cap-drop"); ARG("ALL");
+  ARG("--hostname"); ARG("quixos-restricted");
+  ARG("--clearenv");
+  ARG("--setenv"); ARG("TZ"); ARG("UTC");
+  ARG("--setenv"); ARG("LANG"); ARG("C.UTF-8");
+  FILE *closure = fopen(SANDBOX_CLOSURE, "r");
+  if (!closure)
+    fail("runtime closure");
+  char *line = NULL;
+  size_t capacity = 0;
+  while (getline(&line, &capacity, closure) >= 0) {
+    line[strcspn(line, "\n")] = 0;
+    if (strncmp(line, "/nix/store/", 11) || strchr(line, ' ') || count > 4000)
+      fail("runtime closure entry");
+    char *path = strdup(line);
+    if (!path)
+      fail("runtime closure allocation");
+    ARG("--ro-bind"); ARG(path); ARG(path);
+  }
+  fclose(closure);
+  free(line);
+  ARG("--proc"); ARG("/proc");
+  ARG("--dev"); ARG("/dev");
+  ARG("--tmpfs"); ARG("/tmp");
+  ARG("--bind"); ARG(argv[1]); ARG(argv[1]);
+  ARG("--chdir"); ARG(argv[1]);
+  ARG("--seccomp"); ARG(filter_fd);
+  ARG("--"); ARG(SANDBOX_EXECUTABLE);
+  ARG("serve"); ARG("--experimental"); ARG("config.capnp");
+  ARG(NULL);
+  execv(SANDBOX_BWRAP, args);
+  fail("bubblewrap failed");
+}

restricted-execution/sandbox.nix

@@ -0,0 +1,30 @@
+{
+  pkgs,
+  runtime,
+  executable,
+}:
+let
+  closure = pkgs.closureInfo { rootPaths = [ runtime ]; };
+  filter =
+    pkgs.runCommand "quixos-restricted-seccomp"
+      {
+        nativeBuildInputs = [
+          pkgs.stdenv.cc
+          pkgs.pkg-config
+        ];
+        buildInputs = [ pkgs.libseccomp ];
+      }
+      ''
+        $CC -O2 -Wall -Wextra -Werror ${./seccomp.c} $(pkg-config --cflags --libs libseccomp) -o filter
+        ./filter > "$out"
+      '';
+in
+pkgs.runCommand "quixos-restricted-sandbox" { nativeBuildInputs = [ pkgs.stdenv.cc ]; } ''
+  mkdir -p "$out/bin"
+  $CC -O2 -Wall -Wextra -Werror ${./sandbox.c} \
+    -DSANDBOX_BWRAP='"${pkgs.bubblewrap}/bin/bwrap"' \
+    -DSANDBOX_EXECUTABLE='"${executable}"' \
+    -DSANDBOX_FILTER='"${filter}"' \
+    -DSANDBOX_CLOSURE='"${closure}/store-paths"' \
+    -o "$out/bin/quixos-restricted-sandbox"
+''

restricted-execution/seccomp.c

@@ -0,0 +1,54 @@
+#define _GNU_SOURCE
+#include <errno.h>
+#include <linux/sched.h>
+#include <seccomp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+/* Layered on an empty mount tree, fresh PID/user/network namespaces, no caps,
+ * no inherited descriptors and an immutable runtime closure. Deny kernel and
+ * process escape surfaces; retain V8's ordinary memory/thread syscalls. */
+static void checked(int result) {
+  if (result < 0) {
+    fprintf(stderr, "seccomp rule: %d\n", result);
+    exit(1);
+  }
+}
+int main(void) {
+  scmp_filter_ctx filter = seccomp_init(SCMP_ACT_ALLOW);
+  if (!filter) return 1;
+  const char *denied[] = {
+    "fork", "vfork", "execveat", "ptrace", "process_vm_readv", "process_vm_writev",
+    "pidfd_getfd", "kcmp", "bpf", "perf_event_open", "userfaultfd", "io_uring_setup",
+    "mount", "umount2", "pivot_root", "chroot", "move_mount", "open_tree", "fsopen",
+    "fsconfig", "fsmount", "fspick", "mount_setattr", "open_by_handle_at",
+    "name_to_handle_at", "setns", "unshare", "keyctl", "add_key", "request_key",
+    "reboot", "kexec_load", "kexec_file_load", "init_module", "finit_module",
+    "delete_module", "swapon", "swapoff", "syslog", "acct", "sethostname",
+    "setdomainname", "iopl", "ioperm", "lookup_dcookie", NULL
+  };
+  for (size_t i = 0; denied[i]; i++) {
+    int syscall = seccomp_syscall_resolve_name(denied[i]);
+    if (syscall != __NR_SCMP_ERROR)
+      checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), syscall, 0));
+  }
+  /* glibc falls back to clone when clone3 is unavailable. Only thread creation
+   * is admitted; subprocesses and new namespaces cannot be manufactured. */
+  checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(ENOSYS), SCMP_SYS(clone3), 0));
+  checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(clone), 1,
+    SCMP_A0(SCMP_CMP_MASKED_EQ, CLONE_THREAD, 0)));
+  const unsigned long namespaces[] = {CLONE_NEWUSER, CLONE_NEWNS, CLONE_NEWNET,
+    CLONE_NEWPID, CLONE_NEWIPC, CLONE_NEWUTS, CLONE_NEWCGROUP};
+  for (size_t i = 0; i < sizeof(namespaces) / sizeof(namespaces[0]); i++)
+    checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(clone), 1,
+      SCMP_A0(SCMP_CMP_MASKED_EQ, namespaces[i], namespaces[i])));
+  checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(socket), 1,
+    SCMP_A0(SCMP_CMP_NE, AF_UNIX)));
+  checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(ioctl), 1,
+    SCMP_A1(SCMP_CMP_EQ, 0x5412))); /* TIOCSTI */
+  checked(seccomp_export_bpf(filter, STDOUT_FILENO));
+  seccomp_release(filter);
+  return 0;
+}

restricted-execution/service.mjs

@@ -37,13 +37,11 @@ export class RestrictedEngineBudget {
       let releasing;
       const release = () =>
         (releasing ??= (async () => {
-          try {
+          await resource.release?.();
-            await resource.release?.();
+          this.#leases.delete(lease);
-          } finally {
-            this.#leases.delete(lease);
-          }
         })());
       release.attach = resource.attach;
+      release.usage = resource.usage;
       return release;
     } catch (error) {
       this.#leases.delete(lease);
@@ -83,6 +81,7 @@ export async function serveRestrictedRegistry(config, io = process) {
     !absolute(config.workerd) ||
     !absolute(config.worker) ||
     !absolute(config.module) ||
+    (config.sandbox !== undefined && !absolute(config.sandbox)) ||
     !/^[$A-Z_a-z][$\w]*$/.test(config.registryExport) ||
     !Array.isArray(config.exports) ||
     !config.exports.length ||
@@ -160,19 +159,38 @@ export async function serveRestrictedRegistry(config, io = process) {
       throw fixedError("RESTRICTED_BACKPRESSURE");
     io.stdout.write(line);
   };
-  // Linux USER_HZ is 100 on the supported NixOS architectures. Read process
+  // A sandbox has namespace helpers as well as workerd. Charge the complete
-  // CPU outside the engine; authored code cannot forge these usage messages.
+  // delegated cgroup when available; the qualification fallback follows the
+  // bounded process tree, never trusting a worker's self-reported counters.
+  const engineUsage = async (engine) => {
+    if (engine.release?.usage) return engine.release.usage();
+    engine.processTicks ??= new Map();
+    const pending = [engine.child.pid];
+    let memoryBytes = 0,
+      visited = 0;
+    while (pending.length) {
+      if (++visited > 64) throw fixedError("RESTRICTED_PROCESS_LIMIT");
+      const pid = pending.pop();
+      try {
+        const stat = await readFile(`/proc/${pid}/stat`, "utf8");
+        const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
+        engine.processTicks.set(`${pid}:${fields[19]}`, Number(fields[11]) + Number(fields[12]));
+        const status = await readFile(`/proc/${pid}/status`, "utf8");
+        memoryBytes += Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? 0) * 1024;
+        const children = await readFile(`/proc/${pid}/task/${pid}/children`, "utf8");
+        pending.push(...children.trim().split(/\s+/).filter(Boolean).map(Number));
+      } catch (error) {
+        if (!["ENOENT", "ESRCH"].includes(error.code)) throw error;
+      }
+    }
+    return {
+      cpuMicroseconds: [...engine.processTicks.values()].reduce((sum, ticks) => sum + ticks * 10000, 0),
+      memoryBytes,
+    };
+  };
   const sampleCPU = async (call) => {
     if (!call.child?.pid) return;
-    let text;
+    const { cpuMicroseconds: total } = await engineUsage(call.engine);
-    try {
-      text = await readFile(`/proc/${call.child.pid}/stat`, "utf8");
-    } catch (error) {
-      if (error.code === "ENOENT" || error.code === "ESRCH") return;
-      throw error;
-    }
-    const fields = text.slice(text.lastIndexOf(")") + 2).split(" ");
-    const total = (Number(fields[11]) + Number(fields[12])) * 10000;
     const cpuMicroseconds = total - (call.cpuBaseline ?? 0);
     if (call.engine) call.engine.cpuTotal = total;
     if (!Number.isSafeInteger(cpuMicroseconds) || cpuMicroseconds < (call.cpuMicroseconds ?? 0))
@@ -217,8 +235,7 @@ export async function serveRestrictedRegistry(config, io = process) {
         // A delayed child request can never acquire the next call's authority.
         if (engine.current === call) engine.current = null;
         if (!stopped && reason.code === "INVOCATION_ENDED" && !usageFailure && !engine.destroying) {
-          const status = await readFile(`/proc/${engine.child.pid}/status`, "utf8").catch(() => "");
+          const rss = (await engineUsage(engine)).memoryBytes;
-          const rss = Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? Infinity) * 1024;
           metrics.peakRSSBytes = Math.max(metrics.peakRSSBytes, rss);
           // Bound retained engine heaps as well as lifetime. Hard aggregate
           // physical memory remains the supervising process/cgroup's contract.
@@ -380,19 +397,25 @@ export async function serveRestrictedRegistry(config, io = process) {
     const file = join(engine.directory, "config.capnp");
     await writeFile(file, configuration);
     if (!call.open) return;
-    engine.child = spawn(config.workerd, ["serve", "--experimental", file], {
+    engine.child = spawn(
-      detached: true,
+      config.sandbox ?? config.workerd,
-      env: { TZ: "UTC", LANG: "C.UTF-8" },
+      config.sandbox ? [engine.directory] : ["serve", "--experimental", file],
-      cwd: engine.directory,
+      {
-      stdio: ["ignore", "ignore", "pipe"],
+        detached: true,
-    });
+        env: { TZ: "UTC", LANG: "C.UTF-8" },
+        cwd: engine.directory,
+        stdio: [config.sandbox ? "pipe" : "ignore", "ignore", "pipe"],
+      },
+    );
     metrics.starts++;
     engine.exited = new Promise((resolve) => {
       engine.child.once("exit", resolve);
       engine.child.once("error", resolve);
     });
+    engine.child.stdin?.on("error", () => engine.current?.terminate("RESTRICTED_START_FAILED"));
     call.child = engine.child;
     await engine.release.attach?.(engine.child.pid);
+    if (config.sandbox && call.open) engine.child.stdin.end(Buffer.from([1]));
     watchCPU(call);
     let diagnostics = 0;
     engine.child.stderr.on("data", (chunk) => {
@@ -663,6 +686,7 @@ if (process.argv[1] === new URL(import.meta.url).pathname) {
     ...config,
     workerd: process.env.QUIXOS_RESTRICTED_WORKERD,
     worker: process.env.QUIXOS_RESTRICTED_WORKER,
+    sandbox: process.env.QUIXOS_RESTRICTED_SANDBOX,
     module: join(dirname(process.argv[2]), config.module),
   });
 }

restricted-execution/service.test.mjs

@@ -50,7 +50,16 @@ async function fixture(source, options = {}) {
     }
   };
   const service = await serveRestrictedRegistry(
-    { workerd, worker, module, registryExport: "registry", exports: ["test"], sdk: [], ...options },
+    {
+      workerd,
+      worker,
+      module,
+      sandbox: process.env.QUIXOS_TEST_SANDBOX,
+      registryExport: "registry",
+      exports: ["test"],
+      sdk: [],
+      ...options,
+    },
     io,
   );
   assert.deepEqual(await receive(), { abi, kind: "ready", exports: ["test"] });
@@ -86,6 +95,27 @@ async function fixture(source, options = {}) {
 }
 const registry = (code) => `export const registry={exports:['test'],async invoke(request,channel){${code}}};`;
 
+test("uncertain OS cleanup cannot release engine capacity", async () => {
+  const budget = new RestrictedEngineBudget(1, async () => ({
+    async release() {
+      throw Error("kernel cleanup not confirmed");
+    },
+  }));
+  const release = await budget.acquire(
+    () => false,
+    async () => {},
+  );
+  await assert.rejects(release(), /cleanup not confirmed/);
+  assert.equal(budget.stats().engines, 1);
+  await assert.rejects(
+    budget.acquire(
+      () => false,
+      async () => {},
+    ),
+    { code: "RESTRICTED_ENGINE_LIMIT" },
+  );
+});
+
 test("parallel frames route independently and cancellation cannot retire a neighbor", async () => {
   const f = await fixture(
     registry("return await channel({portId:'read',action:{kind:'state',primitive:'read'},input:request.input});"),