diff --git a/.quixos-subtree-source.json b/.quixos-subtree-source.json index 3e26718..66d4584 100644 --- a/.quixos-subtree-source.json +++ b/.quixos-subtree-source.json @@ -1,7 +1,7 @@ { "version": 1, "sourceRepo": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos", - "sourceCommit": "abc91ab1cbbd61ffd7f0a49dbc5768598ac8745c", + "sourceCommit": "932fe7714761fb8b5db4bd6401357cdff762a428", "sourcePath": "quixos-instance/quixos-nix-helpers", "exportName": "quixos-nix-helpers", "mirrorRemote": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git" diff --git a/PORTABLE_ARTIFACTS.md b/PORTABLE_ARTIFACTS.md index bde2e74..bd9b03d 100644 --- a/PORTABLE_ARTIFACTS.md +++ b/PORTABLE_ARTIFACTS.md @@ -113,8 +113,9 @@ service = { }; ``` -Each invocation gets a fresh engine process. The adapter admits one active frame -and a bounded FIFO; queue wait consumes the deadline. Overflow is an explicit +Each invocation gets a fresh Worker in a reusable engine process. An engine runs +one active frame; the adapter has bounded concurrency and queueing. Queue wait +consumes the deadline. Overflow is an explicit `RESTRICTED_QUEUE_FULL` result. Cancellation kills the engine process group and removes its sockets before acknowledgement. All graph calls inherit the trusted invocation identity; authored code cannot choose another frame or workspace. @@ -127,11 +128,15 @@ parent environment credentials, and unlisted modules are denied. Explicit dates, available. Host-managed HTTP bridges expose only the current frame's declared graph ports. Authored serialization is untrusted protocol input. -This is an inner execution restriction, **not a hardened shared-process sandbox**. -One workspace owns the surrounding VM. Systemd limits the entire package cgroup, -including adapter and child engines; memory exhaustion terminates that group. -No per-isolate memory protection is claimed. A guest user can control its VM, -so authoritative graph grants and credentials must remain outside it. +The packaged runtime uses the platform's Bubblewrap/seccomp launcher. Only the +engine's read-only runtime closure and private broker directory are mounted; +native filesystem/process/network escape probes qualify that boundary. Systemd +limits the whole package in a workspace VM. Shared authority additionally charges +each engine to its own cgroup before launch. No per-isolate memory protection or +Cloudflare hosted side-channel defenses are claimed. See the +[runtime decision](../../docs/RESTRICTED_RUNTIME_DECISION.md#os-process-confinement-implementation) +for the exact boundary and residual risks. A guest user can control its VM, so +authoritative graph grants and credentials must remain outside it. Qualification commands: diff --git a/artifacts.nix b/artifacts.nix index 735df8e..5249ced 100644 --- a/artifacts.nix +++ b/artifacts.nix @@ -96,18 +96,7 @@ let esbuild = "${pkgs.esbuild}/bin/esbuild"; shell = "${pkgs.runtimeShell}"; serviceAdapter = ./portable-service.mjs; - restricted = - if service.restricted or false then - let - executor = import ./restricted-execution { inherit pkgs core; }; - in - { - runtime = "${executor.runtime}/bin/quixos-restricted-registry"; - sdk = "${executor.sdk}/core.mjs"; - limits = service.limits or { }; - } - else - null; + restrictedSDK = "${(import ./restricted-execution { inherit pkgs core; }).sdk}/core.mjs"; } ); compiled = diff --git a/build-artifacts.mjs b/build-artifacts.mjs index 4715d82..6a71be2 100644 --- a/build-artifacts.mjs +++ b/build-artifacts.mjs @@ -128,8 +128,7 @@ for (const { target, mode, settings } of builds) { const bundlePath = mode === "remote" ? "share/quixos/remote/server/module.mjs" : `share/quixos/portable/${target}/module.mjs`; await fs.mkdir(path.dirname(path.join(output, bundlePath)), { recursive: true }); - const restricted = !!config.restricted && target === "server"; - const coreImport = target === "browser" ? coreURL : restricted ? config.restricted.sdk : coreServer; + const coreImport = target === "browser" ? coreURL : coreServer; const metadata = path.join(work, "bundle-meta.json"); measureBuild("portable-bundle", { package: pkg.revision, target, mode }, () => execFileSync( @@ -137,7 +136,7 @@ for (const { target, mode, settings } of builds) { [ entry, "--bundle", - `--platform=${target === "browser" || restricted ? "browser" : "node"}`, + `--platform=${target === "browser" ? "browser" : "node"}`, "--format=esm", "--minify-whitespace", "--target=es2024", @@ -159,7 +158,7 @@ for (const { target, mode, settings } of builds) { (i) => i.external && i.path !== coreImport && - !(target === "server" && !restricted && builtinModules.includes(i.path.replace(/^node:/, ""))), + !(target === "server" && builtinModules.includes(i.path.replace(/^node:/, ""))), ), ) ) @@ -182,6 +181,44 @@ for (const { target, mode, settings } of builds) { files: [await recordFile(bundlePath, "text/javascript"), ...sharedFiles], }; implementations.push({ ...body, contentDigest: compiler.implementationDigest(body) }); + if (target === "server" && mode !== "remote" && exports.length) { + // A closed candidate is not an eligibility claim. The host checks these + // bytes and its exact qualified SDK before admitting it to the sandbox. + // Keep a normal service implementation for unsupported dependencies. + const restrictedPath = "share/quixos/restricted/module.mjs"; + const sdkPath = "share/quixos/restricted/core.mjs"; + const sdkName = "quixos:restricted-sdk/v1"; + await fs.mkdir(path.dirname(path.join(output, restrictedPath)), { recursive: true }); + await fs.copyFile(config.restrictedSDK, path.join(output, sdkPath)); + measureBuild("restricted-bundle", { package: pkg.revision }, () => + execFileSync( + config.esbuild, + [ + entry, + "--bundle", + "--platform=node", + "--format=esm", + "--minify-whitespace", + "--target=es2024", + `--outfile=${path.join(output, restrictedPath)}`, + `--alias:@quixos/camino-replica-core=${sdkName}`, + `--external:${sdkName}`, + ], + { cwd: work, stdio: "inherit" }, + ), + ); + const restricted = { + id: "restricted-module", + target: "server", + compatibility: "quixos-restricted-module-v1", + path: restrictedPath, + ports: "quixos-transaction-ports-v1", + registryExport: settings.registryExport, + exports, + files: [await recordFile(restrictedPath, "text/javascript"), await recordFile(sdkPath, "text/javascript")], + }; + implementations.push({ ...restricted, contentDigest: compiler.implementationDigest(restricted) }); + } } const components = config.group && !config.components diff --git a/build-service.mjs b/build-service.mjs index 57e2335..a913054 100644 --- a/build-service.mjs +++ b/build-service.mjs @@ -13,62 +13,6 @@ export async function buildService({ recordFile, }) { const relative = compiler.artifactPath; - if (config.restricted) { - if (config.service.entry || pkg.exports.some((e) => e.execution.kind === "remote")) - throw Error( - "Restricted services require portable snapshot/transaction exports; remote/custom services keep their VM path", - ); - const modules = implementations.filter((i) => i.target === "server"); - if (modules.length !== 1 || pkg.exports.some((e) => !modules[0].exports.includes(e.id))) - throw Error("Restricted service requires exactly one checked portable registry covering all exports"); - const limits = config.restricted.limits; - if (!limits || typeof limits !== "object" || Array.isArray(limits)) - throw Error("Restricted limits must be an object"); - if (Object.keys(limits).some((key) => !["deadlineMs", "maxBytes", "maxPortCalls", "maxQueued"].includes(key))) - throw Error("Unknown restricted service limit"); - for (const [key, min, max] of [ - ["deadlineMs", 1, 60000], - ["maxBytes", 1024, 4194304], - ["maxPortCalls", 1, 10000], - ["maxQueued", 0, 64], - ]) - if (limits[key] !== undefined && (!Number.isSafeInteger(limits[key]) || limits[key] < min || limits[key] > max)) - throw Error("Invalid restricted service limit " + key); - await fs.mkdir(path.join(output, "bin"), { recursive: true }); - await fs.mkdir(path.join(output, "share/quixos/service"), { recursive: true }); - const configFile = "share/quixos/service/restricted.json"; - const module = modules[0]; - await fs.writeFile( - path.join(output, configFile), - JSON.stringify({ - module: path.relative("share/quixos/service", module.path), - registryExport: module.registryExport, - exports: module.exports, - sdk: [{ name: config.restricted.sdk, path: config.restricted.sdk }], - ...limits, - }), - ); - await fs.writeFile( - path.join(output, "bin/service"), - `#!${config.shell}\nservice_root=\${0%/*}/..\nexec ${config.restricted.runtime} "$service_root/${configFile}"\n`, - { mode: 0o755 }, - ); - const body = { - id: "service", - target: "server", - compatibility: "quixos-service-v1", - path: "bin/service", - ports: "quixos-transaction-ports-v1", - exports: module.exports, - files: [ - await recordFile("bin/service", "text/x-shellscript"), - await recordFile(configFile, "application/json"), - ...module.files, - ], - }; - implementations.unshift({ ...body, contentDigest: compiler.implementationDigest(body) }); - return; - } if (pkg.exports.length || config.service.entry) { await fs.mkdir(path.join(output, "bin"), { recursive: true }); await fs.mkdir(path.join(output, "share/quixos/service"), { recursive: true }); @@ -78,7 +22,10 @@ export async function buildService({ throw Error("Custom service entry must be JavaScript; use a separate typed/language build before packaging"); serviceEntry = path.join(config.source, relative(config.service.entry)); } else { - const modules = implementations.filter((implementation) => implementation.target === "server"); + const modules = implementations.filter( + (implementation) => + implementation.target === "server" && implementation.compatibility === "quixos-js-module-v1", + ); if (pkg.exports.some((entry) => !modules.some((module) => module.exports.includes(entry.id)))) throw Error("Generated service does not cover every package export; provide a custom service"); if (!modules.length) throw Error("Generated service requires a checked registry"); diff --git a/restricted-execution/default.nix b/restricted-execution/default.nix index 383bb93..0d4a413 100644 --- a/restricted-execution/default.nix +++ b/restricted-execution/default.nix @@ -1,6 +1,36 @@ { pkgs, core }: let workerd = import ./workerd.nix { inherit pkgs; }; + sandbox = import ./sandbox.nix { + inherit pkgs; + runtime = workerd; + executable = "${workerd}/bin/workerd"; + }; + nativeProbe = + pkgs.runCommand "quixos-restricted-native-probe" + { + nativeBuildInputs = [ pkgs.stdenv.cc ]; + } + '' + mkdir -p "$out/bin" + $CC -O2 -Wall -Wextra -Werror ${./sandbox-probe.c} -o "$out/bin/probe" + ''; + nativeProbeSandbox = import ./sandbox.nix { + inherit pkgs; + runtime = nativeProbe; + executable = "${nativeProbe}/bin/probe"; + }; + identityLauncher = + pkgs.runCommand "quixos-restricted-identity-launcher" + { + nativeBuildInputs = [ pkgs.stdenv.cc ]; + } + '' + mkdir -p "$out/bin" + $CC -O2 -Wall -Wextra -Werror ${./identity-launcher.c} \ + -DSANDBOX_EXECUTABLE='"${sandbox}/bin/quixos-restricted-sandbox"' \ + -o "$out/bin/quixos-restricted-engine" + ''; # Transaction-port/value adapters do not need the browser's CRDT/Wasm engine. # Keep this surface explicit; it is bundled into checked restricted handlers. sdk = pkgs.runCommand "quixos-restricted-sdk" { nativeBuildInputs = [ pkgs.esbuild ]; } '' @@ -23,6 +53,7 @@ let makeWrapper ${pkgs.nodejs_24}/bin/node "$out/bin/quixos-restricted-registry" \ --add-flags "$out/lib/service.mjs" \ --set QUIXOS_RESTRICTED_WORKERD ${workerd}/bin/workerd \ + --set QUIXOS_RESTRICTED_SANDBOX ${sandbox}/bin/quixos-restricted-sandbox \ --set QUIXOS_RESTRICTED_WORKER "$out/lib/worker.mjs" ''; qualification = @@ -40,6 +71,9 @@ in { inherit workerd + sandbox + identityLauncher + nativeProbeSandbox sdk runtime qualification diff --git a/restricted-execution/identity-launcher.c b/restricted-execution/identity-launcher.c new file mode 100644 index 0000000..296a343 --- /dev/null +++ b/restricted-execution/identity-launcher.c @@ -0,0 +1,88 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Installed only as a narrowly executable NixOS wrapper. This trusted helper + * accepts no executable, environment, filesystem root or arbitrary UID. The + * caller is platform authority, never a workspace guest or authored Worker. */ +static void fail(const char *message) { + fprintf(stderr, "RESTRICTED_IDENTITY: %s\n", message); + exit(125); +} + +int main(int argc, char **argv) { + const char *root = "/run/quixos-restricted-engines/"; + uid_t caller = getuid(); + gid_t group = getgid(); + struct passwd *owner = getpwnam("quixos-graph"); + if (geteuid() != 0) fail("privileged wrapper unavailable"); + if (!owner) fail("authority identity unavailable"); + if (caller != owner->pw_uid) fail("caller identity mismatch"); + if (argc != 3) fail("invalid arguments"); + char *end; + unsigned long number = strtoul(argv[2], &end, 10); + if (!*argv[2] || *end || number < 610000 || number >= 610064) + fail("invalid engine identity"); + uid_t identity = (uid_t)number; + if (strlen(argv[1]) != strlen(root) + 16 || strncmp(argv[1], root, strlen(root))) + fail("invalid engine directory"); + const char *name = argv[1] + strlen(root); + if (strncmp(name, "qx-engine-", 10) || + strlen(name) != 16 || strspn(name + 10, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") != 6) + fail("invalid engine directory"); + int parent = open("/run/quixos-restricted-engines", O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + struct stat info; + if (parent < 0 || fstat(parent, &info) || info.st_uid != caller || (info.st_mode & 066)) + fail("invalid engine root"); + int directory = openat(parent, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + if (directory < 0 || fstat(directory, &info) || info.st_uid != caller || (info.st_mode & 077)) + fail("invalid engine owner"); + DIR *entries = fdopendir(dup(directory)); + if (!entries) fail("engine entries"); + struct dirent *entry; + unsigned int files = 0; + while ((entry = readdir(entries))) { + const char *file = entry->d_name; + if (!strcmp(file, ".") || !strcmp(file, "..")) continue; + int socket = !strcmp(file, "broker.sock"); + if ((!socket && strcmp(file, "config.capnp") && strcmp(file, "parent.mjs")) || + fstatat(directory, file, &info, AT_SYMLINK_NOFOLLOW) || info.st_uid != caller || + info.st_nlink != 1 || (socket ? !S_ISSOCK(info.st_mode) : !S_ISREG(info.st_mode)) || + info.st_size > 4 * 1024 * 1024) + fail("unexpected engine file"); + files++; + } + closedir(entries); + if (files != 3) fail("incomplete engine configuration"); + const char *files_to_transfer[] = {"config.capnp", "parent.mjs", "broker.sock"}; + for (size_t i = 0; i < 3; i++) { + if (fchownat(directory, files_to_transfer[i], identity, group, AT_SYMLINK_NOFOLLOW) || + fchmodat(directory, files_to_transfer[i], 0660, 0)) + fail("engine file ownership"); + } + if (fchown(directory, identity, group) || fchmod(directory, 0770)) + fail("engine directory ownership"); + close(directory); + close(parent); + struct rlimit core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &core) || setgroups(0, NULL) || + setresgid(identity, identity, identity) || setresuid(identity, identity, identity) || + prctl(PR_SET_DUMPABLE, 0) || prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + fail("identity confinement"); + /* The containing directory grants only its owner and authority access. A + * socket created by workerd must also admit authority's different UID/GID. */ + umask(0000); + char *args[] = {SANDBOX_EXECUTABLE, argv[1], NULL}; + clearenv(); + execv(SANDBOX_EXECUTABLE, args); + fail("sandbox launch"); +} diff --git a/restricted-execution/identity-module.nix b/restricted-execution/identity-module.nix new file mode 100644 index 0000000..3aa0ba7 --- /dev/null +++ b/restricted-execution/identity-module.nix @@ -0,0 +1,28 @@ +{ + executor, + enable, + controlGroup, +}: +{ config, lib, ... }: +{ + config = lib.mkIf enable { + assertions = [ + { + assertion = lib.all (user: user.uid == null || user.uid < 610000 || user.uid >= 610064) ( + builtins.attrValues config.users.users + ); + message = "UIDs 610000–610063 are reserved for restricted engine identities."; + } + ]; + systemd.tmpfiles.rules = [ + "d /run/quixos-restricted-engines 0711 quixos-graph ${controlGroup} -" + ]; + security.wrappers.quixos-restricted-engine = { + source = "${executor.identityLauncher}/bin/quixos-restricted-engine"; + owner = "root"; + group = controlGroup; + permissions = "u+rx,g+rx,o-rwx"; + setuid = true; + }; + }; +} diff --git a/restricted-execution/sandbox-probe.c b/restricted-execution/sandbox-probe.c new file mode 100644 index 0000000..a58fcd0 --- /dev/null +++ b/restricted-execution/sandbox-probe.c @@ -0,0 +1,64 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int condition, const char *label) { + if (!condition) { + fprintf(stderr, "ESCAPE: %s (%s)\n", label, strerror(errno)); + exit(1); + } +} +int main(void) { + for (int fd = 3; fd < 256; fd++) + check(fcntl(fd, F_GETFD) < 0 && errno == EBADF, "inherited descriptor"); + check(!getenv("QUIXOS_VM_SECRET"), "parent environment"); + check(open("/etc/test-secret", O_RDONLY) < 0, "authority filesystem"); + check(open("/sys/fs/cgroup/cgroup.procs", O_WRONLY) < 0, "resource controls"); + check(open("/proc/1/root/etc/test-secret", O_RDONLY) < 0, "proc root escape"); + check(socket(AF_INET, SOCK_STREAM, 0) < 0 && errno == EPERM, "IPv4 socket"); + check(socket(AF_INET6, SOCK_STREAM, 0) < 0 && errno == EPERM, "IPv6 socket"); + check(socket(AF_NETLINK, SOCK_RAW, 0) < 0 && errno == EPERM, "netlink socket"); + check(ptrace(PTRACE_TRACEME, 0, NULL, NULL) < 0 && errno == EPERM, "ptrace"); + check(unshare(CLONE_NEWUSER) < 0, "nested user namespace"); + pid_t child = fork(); + if (child == 0) _exit(2); + check(child < 0 && errno == EPERM, "subprocess"); + check(access("/bin/sh", X_OK) < 0, "ambient executable"); + check(getppid() <= 1, "host process visibility"); + struct sockaddr_un address = {.sun_family = AF_UNIX}; + int outside = socket(AF_UNIX, SOCK_STREAM, 0); + check(outside >= 0, "Unix socket support"); + strcpy(address.sun_path, "/run/quixos-native-authority/test.sock"); + check(connect(outside, (struct sockaddr *)&address, sizeof(address)) < 0, + "authority Unix socket"); + close(outside); + outside = socket(AF_UNIX, SOCK_STREAM, 0); + memset(address.sun_path, 0, sizeof(address.sun_path)); + strcpy(address.sun_path + 1, "quixos-native-authority"); + check(connect(outside, (struct sockaddr *)&address, + offsetof(struct sockaddr_un, sun_path) + 1 + strlen(address.sun_path + 1)) < 0, + "authority abstract socket"); + close(outside); + char directory[4096]; + check(getcwd(directory, sizeof(directory)) != NULL, "engine directory"); + check(snprintf(address.sun_path, sizeof(address.sun_path), "%s/broker.sock", directory) < + (int)sizeof(address.sun_path), "broker path length"); + int broker = socket(AF_UNIX, SOCK_STREAM, 0); + check(broker >= 0 && connect(broker, (struct sockaddr *)&address, sizeof(address)) == 0, + "authorized broker"); + check(write(broker, "sandbox-probe", 13) == 13, "broker write"); + close(broker); + puts("RESTRICTED_NATIVE_SANDBOX_PASSED"); + return 0; +} diff --git a/restricted-execution/sandbox.c b/restricted-execution/sandbox.c new file mode 100644 index 0000000..b4ee59b --- /dev/null +++ b/restricted-execution/sandbox.c @@ -0,0 +1,77 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +/* Trusted, single-process admission gate. The supervisor charges this PID to + * its engine cgroup before releasing stdin. All subsequently created namespace + * helpers and threads inherit that charge. No authored code runs here. */ +static void fail(const char *message) { + fprintf(stderr, "RESTRICTED_SANDBOX: %s\n", message); + exit(125); +} + +int main(int argc, char **argv) { + struct stat info; + if (argc != 2 || argv[1][0] != '/' || strstr(argv[1], "/../") || + lstat(argv[1], &info) || !S_ISDIR(info.st_mode) || + info.st_uid != getuid() || (info.st_mode & 007)) + fail("invalid engine directory"); + char gate; + if (read(STDIN_FILENO, &gate, 1) != 1 || gate != 1) + fail("engine admission was not released"); + int input = open("/dev/null", O_RDONLY); + if (input < 0 || dup2(input, STDIN_FILENO) < 0) + fail("stdin"); + close(input); + int filter = open(SANDBOX_FILTER, O_RDONLY); + if (filter < 0) + fail("seccomp filter"); + char filter_fd[32]; + snprintf(filter_fd, sizeof(filter_fd), "%d", filter); + char *args[4096]; + size_t count = 0; +#define ARG(value) args[count++] = (value) + ARG(SANDBOX_BWRAP); + ARG("--unshare-all"); + ARG("--unshare-user"); + ARG("--die-with-parent"); + ARG("--disable-userns"); + ARG("--assert-userns-disabled"); + ARG("--cap-drop"); ARG("ALL"); + ARG("--hostname"); ARG("quixos-restricted"); + ARG("--clearenv"); + ARG("--setenv"); ARG("TZ"); ARG("UTC"); + ARG("--setenv"); ARG("LANG"); ARG("C.UTF-8"); + FILE *closure = fopen(SANDBOX_CLOSURE, "r"); + if (!closure) + fail("runtime closure"); + char *line = NULL; + size_t capacity = 0; + while (getline(&line, &capacity, closure) >= 0) { + line[strcspn(line, "\n")] = 0; + if (strncmp(line, "/nix/store/", 11) || strchr(line, ' ') || count > 4000) + fail("runtime closure entry"); + char *path = strdup(line); + if (!path) + fail("runtime closure allocation"); + ARG("--ro-bind"); ARG(path); ARG(path); + } + fclose(closure); + free(line); + ARG("--proc"); ARG("/proc"); + ARG("--dev"); ARG("/dev"); + ARG("--tmpfs"); ARG("/tmp"); + ARG("--bind"); ARG(argv[1]); ARG(argv[1]); + ARG("--chdir"); ARG(argv[1]); + ARG("--seccomp"); ARG(filter_fd); + ARG("--"); ARG(SANDBOX_EXECUTABLE); + ARG("serve"); ARG("--experimental"); ARG("config.capnp"); + ARG(NULL); + execv(SANDBOX_BWRAP, args); + fail("bubblewrap failed"); +} diff --git a/restricted-execution/sandbox.nix b/restricted-execution/sandbox.nix new file mode 100644 index 0000000..f236989 --- /dev/null +++ b/restricted-execution/sandbox.nix @@ -0,0 +1,30 @@ +{ + pkgs, + runtime, + executable, +}: +let + closure = pkgs.closureInfo { rootPaths = [ runtime ]; }; + filter = + pkgs.runCommand "quixos-restricted-seccomp" + { + nativeBuildInputs = [ + pkgs.stdenv.cc + pkgs.pkg-config + ]; + buildInputs = [ pkgs.libseccomp ]; + } + '' + $CC -O2 -Wall -Wextra -Werror ${./seccomp.c} $(pkg-config --cflags --libs libseccomp) -o filter + ./filter > "$out" + ''; +in +pkgs.runCommand "quixos-restricted-sandbox" { nativeBuildInputs = [ pkgs.stdenv.cc ]; } '' + mkdir -p "$out/bin" + $CC -O2 -Wall -Wextra -Werror ${./sandbox.c} \ + -DSANDBOX_BWRAP='"${pkgs.bubblewrap}/bin/bwrap"' \ + -DSANDBOX_EXECUTABLE='"${executable}"' \ + -DSANDBOX_FILTER='"${filter}"' \ + -DSANDBOX_CLOSURE='"${closure}/store-paths"' \ + -o "$out/bin/quixos-restricted-sandbox" +'' diff --git a/restricted-execution/seccomp.c b/restricted-execution/seccomp.c new file mode 100644 index 0000000..2a3d6bb --- /dev/null +++ b/restricted-execution/seccomp.c @@ -0,0 +1,54 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +/* Layered on an empty mount tree, fresh PID/user/network namespaces, no caps, + * no inherited descriptors and an immutable runtime closure. Deny kernel and + * process escape surfaces; retain V8's ordinary memory/thread syscalls. */ +static void checked(int result) { + if (result < 0) { + fprintf(stderr, "seccomp rule: %d\n", result); + exit(1); + } +} +int main(void) { + scmp_filter_ctx filter = seccomp_init(SCMP_ACT_ALLOW); + if (!filter) return 1; + const char *denied[] = { + "fork", "vfork", "execveat", "ptrace", "process_vm_readv", "process_vm_writev", + "pidfd_getfd", "kcmp", "bpf", "perf_event_open", "userfaultfd", "io_uring_setup", + "mount", "umount2", "pivot_root", "chroot", "move_mount", "open_tree", "fsopen", + "fsconfig", "fsmount", "fspick", "mount_setattr", "open_by_handle_at", + "name_to_handle_at", "setns", "unshare", "keyctl", "add_key", "request_key", + "reboot", "kexec_load", "kexec_file_load", "init_module", "finit_module", + "delete_module", "swapon", "swapoff", "syslog", "acct", "sethostname", + "setdomainname", "iopl", "ioperm", "lookup_dcookie", NULL + }; + for (size_t i = 0; denied[i]; i++) { + int syscall = seccomp_syscall_resolve_name(denied[i]); + if (syscall != __NR_SCMP_ERROR) + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), syscall, 0)); + } + /* glibc falls back to clone when clone3 is unavailable. Only thread creation + * is admitted; subprocesses and new namespaces cannot be manufactured. */ + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(ENOSYS), SCMP_SYS(clone3), 0)); + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(clone), 1, + SCMP_A0(SCMP_CMP_MASKED_EQ, CLONE_THREAD, 0))); + const unsigned long namespaces[] = {CLONE_NEWUSER, CLONE_NEWNS, CLONE_NEWNET, + CLONE_NEWPID, CLONE_NEWIPC, CLONE_NEWUTS, CLONE_NEWCGROUP}; + for (size_t i = 0; i < sizeof(namespaces) / sizeof(namespaces[0]); i++) + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(clone), 1, + SCMP_A0(SCMP_CMP_MASKED_EQ, namespaces[i], namespaces[i]))); + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(socket), 1, + SCMP_A0(SCMP_CMP_NE, AF_UNIX))); + checked(seccomp_rule_add(filter, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(ioctl), 1, + SCMP_A1(SCMP_CMP_EQ, 0x5412))); /* TIOCSTI */ + checked(seccomp_export_bpf(filter, STDOUT_FILENO)); + seccomp_release(filter); + return 0; +} diff --git a/restricted-execution/service.mjs b/restricted-execution/service.mjs index 39eedcd..c8efc7d 100644 --- a/restricted-execution/service.mjs +++ b/restricted-execution/service.mjs @@ -37,13 +37,13 @@ export class RestrictedEngineBudget { let releasing; const release = () => (releasing ??= (async () => { - try { - await resource.release?.(); - } finally { - this.#leases.delete(lease); - } + await resource.release?.(); + this.#leases.delete(lease); })()); release.attach = resource.attach; + release.usage = resource.usage; + release.kill = resource.kill; + release.uid = resource.uid; return release; } catch (error) { this.#leases.delete(lease); @@ -83,6 +83,8 @@ export async function serveRestrictedRegistry(config, io = process) { !absolute(config.workerd) || !absolute(config.worker) || !absolute(config.module) || + (config.sandbox !== undefined && !absolute(config.sandbox)) || + (config.sandboxIdentity !== undefined && typeof config.sandboxIdentity !== "boolean") || !/^[$A-Z_a-z][$\w]*$/.test(config.registryExport) || !Array.isArray(config.exports) || !config.exports.length || @@ -160,19 +162,38 @@ export async function serveRestrictedRegistry(config, io = process) { throw fixedError("RESTRICTED_BACKPRESSURE"); io.stdout.write(line); }; - // Linux USER_HZ is 100 on the supported NixOS architectures. Read process - // CPU outside the engine; authored code cannot forge these usage messages. + // A sandbox has namespace helpers as well as workerd. Charge the complete + // delegated cgroup when available; the qualification fallback follows the + // bounded process tree, never trusting a worker's self-reported counters. + const engineUsage = async (engine) => { + if (engine.release?.usage) return engine.release.usage(); + engine.processTicks ??= new Map(); + const pending = [engine.child.pid]; + let memoryBytes = 0, + visited = 0; + while (pending.length) { + if (++visited > 64) throw fixedError("RESTRICTED_PROCESS_LIMIT"); + const pid = pending.pop(); + try { + const stat = await readFile(`/proc/${pid}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + engine.processTicks.set(`${pid}:${fields[19]}`, Number(fields[11]) + Number(fields[12])); + const status = await readFile(`/proc/${pid}/status`, "utf8"); + memoryBytes += Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? 0) * 1024; + const children = await readFile(`/proc/${pid}/task/${pid}/children`, "utf8"); + pending.push(...children.trim().split(/\s+/).filter(Boolean).map(Number)); + } catch (error) { + if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; + } + } + return { + cpuMicroseconds: [...engine.processTicks.values()].reduce((sum, ticks) => sum + ticks * 10000, 0), + memoryBytes, + }; + }; const sampleCPU = async (call) => { if (!call.child?.pid) return; - let text; - try { - text = await readFile(`/proc/${call.child.pid}/stat`, "utf8"); - } catch (error) { - if (error.code === "ENOENT" || error.code === "ESRCH") return; - throw error; - } - const fields = text.slice(text.lastIndexOf(")") + 2).split(" "); - const total = (Number(fields[11]) + Number(fields[12])) * 10000; + const { cpuMicroseconds: total } = await engineUsage(call.engine); const cpuMicroseconds = total - (call.cpuBaseline ?? 0); if (call.engine) call.engine.cpuTotal = total; if (!Number.isSafeInteger(cpuMicroseconds) || cpuMicroseconds < (call.cpuMicroseconds ?? 0)) @@ -217,8 +238,7 @@ export async function serveRestrictedRegistry(config, io = process) { // A delayed child request can never acquire the next call's authority. if (engine.current === call) engine.current = null; if (!stopped && reason.code === "INVOCATION_ENDED" && !usageFailure && !engine.destroying) { - const status = await readFile(`/proc/${engine.child.pid}/status`, "utf8").catch(() => ""); - const rss = Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? Infinity) * 1024; + const rss = (await engineUsage(engine)).memoryBytes; metrics.peakRSSBytes = Math.max(metrics.peakRSSBytes, rss); // Bound retained engine heaps as well as lifetime. Hard aggregate // physical memory remains the supervising process/cgroup's contract. @@ -263,10 +283,13 @@ export async function serveRestrictedRegistry(config, io = process) { if (index >= 0) idleEngines.splice(index, 1); engine.current = null; if (engine.child?.pid) { - try { - process.kill(-engine.child.pid, "SIGKILL"); - } catch (error) { - if (error.code !== "ESRCH") throw error; + if (engine.release.kill) await engine.release.kill(); + else { + try { + process.kill(-engine.child.pid, "SIGKILL"); + } catch (error) { + if (error.code !== "ESRCH") throw error; + } } await engine.exited; } @@ -380,27 +403,37 @@ export async function serveRestrictedRegistry(config, io = process) { const file = join(engine.directory, "config.capnp"); await writeFile(file, configuration); if (!call.open) return; - engine.child = spawn(config.workerd, ["serve", "--experimental", file], { - detached: true, - env: { TZ: "UTC", LANG: "C.UTF-8" }, - cwd: engine.directory, - stdio: ["ignore", "ignore", "pipe"], - }); + if (config.sandboxIdentity && !Number.isSafeInteger(engine.release.uid)) + throw fixedError("RESTRICTED_IDENTITY_UNAVAILABLE"); + engine.child = spawn( + config.sandbox ?? config.workerd, + config.sandbox + ? [engine.directory, ...(config.sandboxIdentity ? [String(engine.release.uid)] : [])] + : ["serve", "--experimental", file], + { + detached: true, + env: { TZ: "UTC", LANG: "C.UTF-8" }, + cwd: engine.directory, + stdio: [config.sandbox ? "pipe" : "ignore", "ignore", "pipe"], + }, + ); metrics.starts++; engine.exited = new Promise((resolve) => { engine.child.once("exit", resolve); engine.child.once("error", resolve); }); + engine.child.stdin?.on("error", () => engine.current?.terminate("RESTRICTED_START_FAILED")); call.child = engine.child; - await engine.release.attach?.(engine.child.pid); - watchCPU(call); let diagnostics = 0; engine.child.stderr.on("data", (chunk) => { if ((diagnostics += chunk.length) > 65536) { if (engine.current?.open) engine.current.terminate("RESTRICTED_DIAGNOSTIC_LIMIT"); else void destroyEngine(engine).catch(fail); - } + } else config.diagnostics?.(chunk); }); + await engine.release.attach?.(engine.child.pid); + if (config.sandbox && call.open) engine.child.stdin.end(Buffer.from([1])); + watchCPU(call); engine.child.once("error", () => engine.current?.terminate("RESTRICTED_START_FAILED")); void engine.exited.then(() => { if (engine.current?.open) engine.current.terminate("RESTRICTED_ENGINE_EXITED"); @@ -512,6 +545,7 @@ export async function serveRestrictedRegistry(config, io = process) { await finish(call, fixedError("INVOCATION_ENDED"), reply); metrics.completed++; } catch (error) { + config.diagnostics?.(Buffer.from(`RESTRICTED_HOST_ERROR:${error.code ?? error.name}\n`)); if (call.open) { const code = /^RESTRICTED_[A-Z_]+$/.test(error.code ?? "") ? error.code : "RESTRICTED_PACKAGE_FAILED"; await finish(call, error, { invocation: message.invocation, status: "failed", error: { code, message: code } }); @@ -663,6 +697,7 @@ if (process.argv[1] === new URL(import.meta.url).pathname) { ...config, workerd: process.env.QUIXOS_RESTRICTED_WORKERD, worker: process.env.QUIXOS_RESTRICTED_WORKER, + sandbox: process.env.QUIXOS_RESTRICTED_SANDBOX, module: join(dirname(process.argv[2]), config.module), }); } diff --git a/restricted-execution/service.test.mjs b/restricted-execution/service.test.mjs index a3c03f5..b9c258d 100644 --- a/restricted-execution/service.test.mjs +++ b/restricted-execution/service.test.mjs @@ -50,7 +50,16 @@ async function fixture(source, options = {}) { } }; const service = await serveRestrictedRegistry( - { workerd, worker, module, registryExport: "registry", exports: ["test"], sdk: [], ...options }, + { + workerd, + worker, + module, + sandbox: process.env.QUIXOS_TEST_SANDBOX, + registryExport: "registry", + exports: ["test"], + sdk: [], + ...options, + }, io, ); assert.deepEqual(await receive(), { abi, kind: "ready", exports: ["test"] }); @@ -86,6 +95,27 @@ async function fixture(source, options = {}) { } const registry = (code) => `export const registry={exports:['test'],async invoke(request,channel){${code}}};`; +test("uncertain OS cleanup cannot release engine capacity", async () => { + const budget = new RestrictedEngineBudget(1, async () => ({ + async release() { + throw Error("kernel cleanup not confirmed"); + }, + })); + const release = await budget.acquire( + () => false, + async () => {}, + ); + await assert.rejects(release(), /cleanup not confirmed/); + assert.equal(budget.stats().engines, 1); + await assert.rejects( + budget.acquire( + () => false, + async () => {}, + ), + { code: "RESTRICTED_ENGINE_LIMIT" }, + ); +}); + test("parallel frames route independently and cancellation cannot retire a neighbor", async () => { const f = await fixture( registry("return await channel({portId:'read',action:{kind:'state',primitive:'read'},input:request.input});"), @@ -407,15 +437,21 @@ test("bounded FIFO rejects overload without losing active or queued frames", asy test("queue wait consumes the deadline and cancelled work never reaches its ports", async () => { const f = await fixture( registry("return await channel({portId:'read',action:{kind:'state',primitive:'read'},input:null});"), - { deadlineMs: 350, maxActive: 1 }, + { deadlineMs: 4000, maxActive: 1 }, ); try { const first = f.start(); - await f.receive(); - const queued = f.start(); - const replies = [await f.receive(), await f.receive()]; - assert.deepEqual(new Set(replies.map((x) => x.invocation)), new Set([first.invocation, queued.invocation])); - assert.ok(replies.every((x) => x.error.code === "RESTRICTED_DEADLINE")); + assert.equal((await f.receive()).kind, "port"); + // Expire while the first call is deliberately blocked. Two equal relative + // deadlines let fast startup legitimately begin the queued call first. + const queued = f.start(null, { root: "queued-deadline", deadlineMs: 50, depth: 1, calls: 10 }); + const expired = await f.receive(); + assert.equal(expired.invocation, queued.invocation); + assert.equal(expired.error.code, "RESTRICTED_DEADLINE"); + f.send({ kind: "cancel", ...first }); + const cancelled = await f.receive(); + assert.equal(cancelled.invocation, first.invocation); + assert.equal(cancelled.kind, "cancelled"); const fresh = f.start(); const port = await f.receive(); assert.equal(port.invocation, fresh.invocation); diff --git a/restricted-execution/workerd.nix b/restricted-execution/workerd.nix index 35a1014..7b25035 100644 --- a/restricted-execution/workerd.nix +++ b/restricted-execution/workerd.nix @@ -1,6 +1,6 @@ # Pinned upstream engine. This binary supplies an import/capability boundary; -# production execution additionally requires OS limits and platform approval -# for shared placement, or a workspace VM for unreviewed application code. +# production execution additionally requires the qualified OS identity, +# namespace, syscall, cgroup and invocation-broker boundaries. { pkgs }: let version = "1.20261005.1";