diff --git a/PORTABLE_ARTIFACTS.md b/PORTABLE_ARTIFACTS.md index bde2e74..bd9b03d 100644 --- a/PORTABLE_ARTIFACTS.md +++ b/PORTABLE_ARTIFACTS.md @@ -113,8 +113,9 @@ service = { }; ``` -Each invocation gets a fresh engine process. The adapter admits one active frame -and a bounded FIFO; queue wait consumes the deadline. Overflow is an explicit +Each invocation gets a fresh Worker in a reusable engine process. An engine runs +one active frame; the adapter has bounded concurrency and queueing. Queue wait +consumes the deadline. Overflow is an explicit `RESTRICTED_QUEUE_FULL` result. Cancellation kills the engine process group and removes its sockets before acknowledgement. All graph calls inherit the trusted invocation identity; authored code cannot choose another frame or workspace. @@ -127,11 +128,15 @@ parent environment credentials, and unlisted modules are denied. Explicit dates, available. Host-managed HTTP bridges expose only the current frame's declared graph ports. Authored serialization is untrusted protocol input. -This is an inner execution restriction, **not a hardened shared-process sandbox**. -One workspace owns the surrounding VM. Systemd limits the entire package cgroup, -including adapter and child engines; memory exhaustion terminates that group. -No per-isolate memory protection is claimed. A guest user can control its VM, -so authoritative graph grants and credentials must remain outside it. +The packaged runtime uses the platform's Bubblewrap/seccomp launcher. Only the +engine's read-only runtime closure and private broker directory are mounted; +native filesystem/process/network escape probes qualify that boundary. Systemd +limits the whole package in a workspace VM. Shared authority additionally charges +each engine to its own cgroup before launch. No per-isolate memory protection or +Cloudflare hosted side-channel defenses are claimed. See the +[runtime decision](../../docs/RESTRICTED_RUNTIME_DECISION.md#os-process-confinement-implementation) +for the exact boundary and residual risks. A guest user can control its VM, so +authoritative graph grants and credentials must remain outside it. Qualification commands: diff --git a/restricted-execution/default.nix b/restricted-execution/default.nix index 383bb93..4d12450 100644 --- a/restricted-execution/default.nix +++ b/restricted-execution/default.nix @@ -1,6 +1,25 @@ { pkgs, core }: let workerd = import ./workerd.nix { inherit pkgs; }; + sandbox = import ./sandbox.nix { + inherit pkgs; + runtime = workerd; + executable = "${workerd}/bin/workerd"; + }; + nativeProbe = + pkgs.runCommand "quixos-restricted-native-probe" + { + nativeBuildInputs = [ pkgs.stdenv.cc ]; + } + '' + mkdir -p "$out/bin" + $CC -O2 -Wall -Wextra -Werror ${./sandbox-probe.c} -o "$out/bin/probe" + ''; + nativeProbeSandbox = import ./sandbox.nix { + inherit pkgs; + runtime = nativeProbe; + executable = "${nativeProbe}/bin/probe"; + }; # Transaction-port/value adapters do not need the browser's CRDT/Wasm engine. # Keep this surface explicit; it is bundled into checked restricted handlers. sdk = pkgs.runCommand "quixos-restricted-sdk" { nativeBuildInputs = [ pkgs.esbuild ]; } '' @@ -23,6 +42,7 @@ let makeWrapper ${pkgs.nodejs_24}/bin/node "$out/bin/quixos-restricted-registry" \ --add-flags "$out/lib/service.mjs" \ --set QUIXOS_RESTRICTED_WORKERD ${workerd}/bin/workerd \ + --set QUIXOS_RESTRICTED_SANDBOX ${sandbox}/bin/quixos-restricted-sandbox \ --set QUIXOS_RESTRICTED_WORKER "$out/lib/worker.mjs" ''; qualification = @@ -40,6 +60,8 @@ in { inherit workerd + sandbox + nativeProbeSandbox sdk runtime qualification diff --git a/restricted-execution/sandbox-probe.c b/restricted-execution/sandbox-probe.c new file mode 100644 index 0000000..a58fcd0 --- /dev/null +++ b/restricted-execution/sandbox-probe.c @@ -0,0 +1,64 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int condition, const char *label) { + if (!condition) { + fprintf(stderr, "ESCAPE: %s (%s)\n", label, strerror(errno)); + exit(1); + } +} +int main(void) { + for (int fd = 3; fd < 256; fd++) + check(fcntl(fd, F_GETFD) < 0 && errno == EBADF, "inherited descriptor"); + check(!getenv("QUIXOS_VM_SECRET"), "parent environment"); + check(open("/etc/test-secret", O_RDONLY) < 0, "authority filesystem"); + check(open("/sys/fs/cgroup/cgroup.procs", O_WRONLY) < 0, "resource controls"); + check(open("/proc/1/root/etc/test-secret", O_RDONLY) < 0, "proc root escape"); + check(socket(AF_INET, SOCK_STREAM, 0) < 0 && errno == EPERM, "IPv4 socket"); + check(socket(AF_INET6, SOCK_STREAM, 0) < 0 && errno == EPERM, "IPv6 socket"); + check(socket(AF_NETLINK, SOCK_RAW, 0) < 0 && errno == EPERM, "netlink socket"); + check(ptrace(PTRACE_TRACEME, 0, NULL, NULL) < 0 && errno == EPERM, "ptrace"); + check(unshare(CLONE_NEWUSER) < 0, "nested user namespace"); + pid_t child = fork(); + if (child == 0) _exit(2); + check(child < 0 && errno == EPERM, "subprocess"); + check(access("/bin/sh", X_OK) < 0, "ambient executable"); + check(getppid() <= 1, "host process visibility"); + struct sockaddr_un address = {.sun_family = AF_UNIX}; + int outside = socket(AF_UNIX, SOCK_STREAM, 0); + check(outside >= 0, "Unix socket support"); + strcpy(address.sun_path, "/run/quixos-native-authority/test.sock"); + check(connect(outside, (struct sockaddr *)&address, sizeof(address)) < 0, + "authority Unix socket"); + close(outside); + outside = socket(AF_UNIX, SOCK_STREAM, 0); + memset(address.sun_path, 0, sizeof(address.sun_path)); + strcpy(address.sun_path + 1, "quixos-native-authority"); + check(connect(outside, (struct sockaddr *)&address, + offsetof(struct sockaddr_un, sun_path) + 1 + strlen(address.sun_path + 1)) < 0, + "authority abstract socket"); + close(outside); + char directory[4096]; + check(getcwd(directory, sizeof(directory)) != NULL, "engine directory"); + check(snprintf(address.sun_path, sizeof(address.sun_path), "%s/broker.sock", directory) < + (int)sizeof(address.sun_path), "broker path length"); + int broker = socket(AF_UNIX, SOCK_STREAM, 0); + check(broker >= 0 && connect(broker, (struct sockaddr *)&address, sizeof(address)) == 0, + "authorized broker"); + check(write(broker, "sandbox-probe", 13) == 13, "broker write"); + close(broker); + puts("RESTRICTED_NATIVE_SANDBOX_PASSED"); + return 0; +} diff --git a/restricted-execution/sandbox.c b/restricted-execution/sandbox.c new file mode 100644 index 0000000..775e5e1 --- /dev/null +++ b/restricted-execution/sandbox.c @@ -0,0 +1,77 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +/* Trusted, single-process admission gate. The supervisor charges this PID to + * its engine cgroup before releasing stdin. All subsequently created namespace + * helpers and threads inherit that charge. No authored code runs here. */ +static void fail(const char *message) { + fprintf(stderr, "RESTRICTED_SANDBOX: %s\n", message); + exit(125); +} + +int main(int argc, char **argv) { + struct stat info; + if (argc != 2 || argv[1][0] != '/' || strstr(argv[1], "/../") || + lstat(argv[1], &info) || !S_ISDIR(info.st_mode) || + info.st_uid != getuid() || (info.st_mode & 077)) + fail("invalid engine directory"); + char gate; + if (read(STDIN_FILENO, &gate, 1) != 1 || gate != 1) + fail("engine admission was not released"); + int input = open("/dev/null", O_RDONLY); + if (input < 0 || dup2(input, STDIN_FILENO) < 0) + fail("stdin"); + close(input); + int filter = open(SANDBOX_FILTER, O_RDONLY); + if (filter < 0) + fail("seccomp filter"); + char filter_fd[32]; + snprintf(filter_fd, sizeof(filter_fd), "%d", filter); + char *args[4096]; + size_t count = 0; +#define ARG(value) args[count++] = (value) + ARG(SANDBOX_BWRAP); + ARG("--unshare-all"); + ARG("--unshare-user"); + ARG("--die-with-parent"); + ARG("--disable-userns"); + ARG("--assert-userns-disabled"); + ARG("--cap-drop"); ARG("ALL"); + ARG("--hostname"); ARG("quixos-restricted"); + ARG("--clearenv"); + ARG("--setenv"); ARG("TZ"); ARG("UTC"); + ARG("--setenv"); ARG("LANG"); ARG("C.UTF-8"); + FILE *closure = fopen(SANDBOX_CLOSURE, "r"); + if (!closure) + fail("runtime closure"); + char *line = NULL; + size_t capacity = 0; + while (getline(&line, &capacity, closure) >= 0) { + line[strcspn(line, "\n")] = 0; + if (strncmp(line, "/nix/store/", 11) || strchr(line, ' ') || count > 4000) + fail("runtime closure entry"); + char *path = strdup(line); + if (!path) + fail("runtime closure allocation"); + ARG("--ro-bind"); ARG(path); ARG(path); + } + fclose(closure); + free(line); + ARG("--proc"); ARG("/proc"); + ARG("--dev"); ARG("/dev"); + ARG("--tmpfs"); ARG("/tmp"); + ARG("--bind"); ARG(argv[1]); ARG(argv[1]); + ARG("--chdir"); ARG(argv[1]); + ARG("--seccomp"); ARG(filter_fd); + ARG("--"); ARG(SANDBOX_EXECUTABLE); + ARG("serve"); ARG("--experimental"); ARG("config.capnp"); + ARG(NULL); + execv(SANDBOX_BWRAP, args); + fail("bubblewrap failed"); +} diff --git a/restricted-execution/sandbox.nix b/restricted-execution/sandbox.nix new file mode 100644 index 0000000..f236989 --- /dev/null +++ b/restricted-execution/sandbox.nix @@ -0,0 +1,30 @@ +{ + pkgs, + runtime, + executable, +}: +let + closure = pkgs.closureInfo { rootPaths = [ runtime ]; }; + filter = + pkgs.runCommand "quixos-restricted-seccomp" + { + nativeBuildInputs = [ + pkgs.stdenv.cc + pkgs.pkg-config + ]; + buildInputs = [ pkgs.libseccomp ]; + } + '' + $CC -O2 -Wall -Wextra -Werror ${./seccomp.c} $(pkg-config --cflags --libs libseccomp) -o filter + ./filter > "$out" + ''; +in +pkgs.runCommand "quixos-restricted-sandbox" { nativeBuildInputs = [ pkgs.stdenv.cc ]; } '' + mkdir -p "$out/bin" + $CC -O2 -Wall -Wextra -Werror ${./sandbox.c} \ + -DSANDBOX_BWRAP='"${pkgs.bubblewrap}/bin/bwrap"' \ + -DSANDBOX_EXECUTABLE='"${executable}"' \ + -DSANDBOX_FILTER='"${filter}"' \ + -DSANDBOX_CLOSURE='"${closure}/store-paths"' \ + -o "$out/bin/quixos-restricted-sandbox" +'' diff --git a/restricted-execution/seccomp.c b/restricted-execution/seccomp.c new file mode 100644 index 0000000..2a3d6bb --- /dev/null +++ b/restricted-execution/seccomp.c @@ -0,0 +1,54 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +/* Layered on an empty mount tree, fresh PID/user/network namespaces, no caps, + * no inherited descriptors and an immutable runtime closure. Deny kernel and + * process escape surfaces; retain V8's ordinary memory/thread syscalls. */ +static void checked(int result) { + if (result < 0) { + fprintf(stderr, "seccomp rule: %d\n", result); + exit(1); + } +} +int main(void) { + scmp_filter_ctx filter = seccomp_init(SCMP_ACT_ALLOW); + if (!filter) return 1; + const char *denied[] = { + "fork", "vfork", "execveat", "ptrace", "process_vm_readv", "process_vm_writev", + "pidfd_getfd", "kcmp", "bpf", "perf_event_open", "userfaultfd", "io_uring_setup", + "mount", "umount2", "pivot_root", "chroot", "move_mount", "open_tree", "fsopen", + "fsconfig", "fsmount", "fspick", "mount_setattr", "open_by_handle_at", + "name_to_handle_at", "setns", "unshare", "keyctl", "add_key", "request_key", + "reboot", "kexec_load", "kexec_file_load", "init_module", "finit_module", + "delete_module", "swapon", "swapoff", "syslog", "acct", "sethostname", + "setdomainname", "iopl", "ioperm", "lookup_dcookie", NULL + }; + for (size_t i = 0; denied[i]; i++) { + int syscall = seccomp_syscall_resolve_name(denied[i]); + if (syscall != __NR_SCMP_ERROR) + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), syscall, 0)); + } + /* glibc falls back to clone when clone3 is unavailable. Only thread creation + * is admitted; subprocesses and new namespaces cannot be manufactured. */ + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(ENOSYS), SCMP_SYS(clone3), 0)); + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(clone), 1, + SCMP_A0(SCMP_CMP_MASKED_EQ, CLONE_THREAD, 0))); + const unsigned long namespaces[] = {CLONE_NEWUSER, CLONE_NEWNS, CLONE_NEWNET, + CLONE_NEWPID, CLONE_NEWIPC, CLONE_NEWUTS, CLONE_NEWCGROUP}; + for (size_t i = 0; i < sizeof(namespaces) / sizeof(namespaces[0]); i++) + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(clone), 1, + SCMP_A0(SCMP_CMP_MASKED_EQ, namespaces[i], namespaces[i]))); + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(socket), 1, + SCMP_A0(SCMP_CMP_NE, AF_UNIX))); + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(ioctl), 1, + SCMP_A1(SCMP_CMP_EQ, 0x5412))); /* TIOCSTI */ + checked(seccomp_export_bpf(filter, STDOUT_FILENO)); + seccomp_release(filter); + return 0; +} diff --git a/restricted-execution/service.mjs b/restricted-execution/service.mjs index 39eedcd..b3de72b 100644 --- a/restricted-execution/service.mjs +++ b/restricted-execution/service.mjs @@ -37,13 +37,11 @@ export class RestrictedEngineBudget { let releasing; const release = () => (releasing ??= (async () => { - try { - await resource.release?.(); - } finally { - this.#leases.delete(lease); - } + await resource.release?.(); + this.#leases.delete(lease); })()); release.attach = resource.attach; + release.usage = resource.usage; return release; } catch (error) { this.#leases.delete(lease); @@ -83,6 +81,7 @@ export async function serveRestrictedRegistry(config, io = process) { !absolute(config.workerd) || !absolute(config.worker) || !absolute(config.module) || + (config.sandbox !== undefined && !absolute(config.sandbox)) || !/^[$A-Z_a-z][$\w]*$/.test(config.registryExport) || !Array.isArray(config.exports) || !config.exports.length || @@ -160,19 +159,38 @@ export async function serveRestrictedRegistry(config, io = process) { throw fixedError("RESTRICTED_BACKPRESSURE"); io.stdout.write(line); }; - // Linux USER_HZ is 100 on the supported NixOS architectures. Read process - // CPU outside the engine; authored code cannot forge these usage messages. + // A sandbox has namespace helpers as well as workerd. Charge the complete + // delegated cgroup when available; the qualification fallback follows the + // bounded process tree, never trusting a worker's self-reported counters. + const engineUsage = async (engine) => { + if (engine.release?.usage) return engine.release.usage(); + engine.processTicks ??= new Map(); + const pending = [engine.child.pid]; + let memoryBytes = 0, + visited = 0; + while (pending.length) { + if (++visited > 64) throw fixedError("RESTRICTED_PROCESS_LIMIT"); + const pid = pending.pop(); + try { + const stat = await readFile(`/proc/${pid}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + engine.processTicks.set(`${pid}:${fields[19]}`, Number(fields[11]) + Number(fields[12])); + const status = await readFile(`/proc/${pid}/status`, "utf8"); + memoryBytes += Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? 0) * 1024; + const children = await readFile(`/proc/${pid}/task/${pid}/children`, "utf8"); + pending.push(...children.trim().split(/\s+/).filter(Boolean).map(Number)); + } catch (error) { + if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; + } + } + return { + cpuMicroseconds: [...engine.processTicks.values()].reduce((sum, ticks) => sum + ticks * 10000, 0), + memoryBytes, + }; + }; const sampleCPU = async (call) => { if (!call.child?.pid) return; - let text; - try { - text = await readFile(`/proc/${call.child.pid}/stat`, "utf8"); - } catch (error) { - if (error.code === "ENOENT" || error.code === "ESRCH") return; - throw error; - } - const fields = text.slice(text.lastIndexOf(")") + 2).split(" "); - const total = (Number(fields[11]) + Number(fields[12])) * 10000; + const { cpuMicroseconds: total } = await engineUsage(call.engine); const cpuMicroseconds = total - (call.cpuBaseline ?? 0); if (call.engine) call.engine.cpuTotal = total; if (!Number.isSafeInteger(cpuMicroseconds) || cpuMicroseconds < (call.cpuMicroseconds ?? 0)) @@ -217,8 +235,7 @@ export async function serveRestrictedRegistry(config, io = process) { // A delayed child request can never acquire the next call's authority. if (engine.current === call) engine.current = null; if (!stopped && reason.code === "INVOCATION_ENDED" && !usageFailure && !engine.destroying) { - const status = await readFile(`/proc/${engine.child.pid}/status`, "utf8").catch(() => ""); - const rss = Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? Infinity) * 1024; + const rss = (await engineUsage(engine)).memoryBytes; metrics.peakRSSBytes = Math.max(metrics.peakRSSBytes, rss); // Bound retained engine heaps as well as lifetime. Hard aggregate // physical memory remains the supervising process/cgroup's contract. @@ -380,19 +397,25 @@ export async function serveRestrictedRegistry(config, io = process) { const file = join(engine.directory, "config.capnp"); await writeFile(file, configuration); if (!call.open) return; - engine.child = spawn(config.workerd, ["serve", "--experimental", file], { - detached: true, - env: { TZ: "UTC", LANG: "C.UTF-8" }, - cwd: engine.directory, - stdio: ["ignore", "ignore", "pipe"], - }); + engine.child = spawn( + config.sandbox ?? config.workerd, + config.sandbox ? [engine.directory] : ["serve", "--experimental", file], + { + detached: true, + env: { TZ: "UTC", LANG: "C.UTF-8" }, + cwd: engine.directory, + stdio: [config.sandbox ? "pipe" : "ignore", "ignore", "pipe"], + }, + ); metrics.starts++; engine.exited = new Promise((resolve) => { engine.child.once("exit", resolve); engine.child.once("error", resolve); }); + engine.child.stdin?.on("error", () => engine.current?.terminate("RESTRICTED_START_FAILED")); call.child = engine.child; await engine.release.attach?.(engine.child.pid); + if (config.sandbox && call.open) engine.child.stdin.end(Buffer.from([1])); watchCPU(call); let diagnostics = 0; engine.child.stderr.on("data", (chunk) => { @@ -663,6 +686,7 @@ if (process.argv[1] === new URL(import.meta.url).pathname) { ...config, workerd: process.env.QUIXOS_RESTRICTED_WORKERD, worker: process.env.QUIXOS_RESTRICTED_WORKER, + sandbox: process.env.QUIXOS_RESTRICTED_SANDBOX, module: join(dirname(process.argv[2]), config.module), }); } diff --git a/restricted-execution/service.test.mjs b/restricted-execution/service.test.mjs index a3c03f5..4cbdbed 100644 --- a/restricted-execution/service.test.mjs +++ b/restricted-execution/service.test.mjs @@ -50,7 +50,16 @@ async function fixture(source, options = {}) { } }; const service = await serveRestrictedRegistry( - { workerd, worker, module, registryExport: "registry", exports: ["test"], sdk: [], ...options }, + { + workerd, + worker, + module, + sandbox: process.env.QUIXOS_TEST_SANDBOX, + registryExport: "registry", + exports: ["test"], + sdk: [], + ...options, + }, io, ); assert.deepEqual(await receive(), { abi, kind: "ready", exports: ["test"] }); @@ -86,6 +95,27 @@ async function fixture(source, options = {}) { } const registry = (code) => `export const registry={exports:['test'],async invoke(request,channel){${code}}};`; +test("uncertain OS cleanup cannot release engine capacity", async () => { + const budget = new RestrictedEngineBudget(1, async () => ({ + async release() { + throw Error("kernel cleanup not confirmed"); + }, + })); + const release = await budget.acquire( + () => false, + async () => {}, + ); + await assert.rejects(release(), /cleanup not confirmed/); + assert.equal(budget.stats().engines, 1); + await assert.rejects( + budget.acquire( + () => false, + async () => {}, + ), + { code: "RESTRICTED_ENGINE_LIMIT" }, + ); +}); + test("parallel frames route independently and cancellation cannot retire a neighbor", async () => { const f = await fixture( registry("return await channel({portId:'read',action:{kind:'state',primitive:'read'},input:request.input});"),