Compare commits

...
10 changed files with 352 additions and 48 deletions
+1 -1
View File
@@ -1,7 +1,7 @@
{
"version": 1,
"sourceRepo": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos",
"sourceCommit": "c4319441699b7b8c5af7e70dade6e4ad8cccddea",
"sourceCommit": "286ada5419c4fc2184b387cea4715fae1a74bde5",
"sourcePath": "quixos-protocol",
"exportName": "quixos-protocol",
"mirrorRemote": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-protocol.git"
+11 -5
View File
@@ -1,5 +1,11 @@
# Closed execution plans and artifacts
This page documents current compiler/runtime representations. Its universal
artifact inventory, React metadata and overlapping world/installation structures
are [known architectural gaps](../../../docs/ARCHITECTURE_STATUS.md), not the
[core package contract](../../../docs/ARCHITECTURE.md). Historical LF task numbers
refer to the original implementation sequence, not an active queue.
`compileExecutionPlan` validates a workspace, links its checked queries, resolves
public forwarding, and emits one versioned plan with an SHA-256 digest. It retains
exact interface/package revisions and generic applications, owner initialization,
@@ -60,11 +66,11 @@ the core codec's canonical ordering and domain-separated SHA-256 framing. The
monorepo `camino-execution-world` check compares the compiler and core implementations.
No machine-local Nix path enters the world descriptor.
The current binding schema is the closed workspace schema. This is conservative:
an unrelated contract change can require checking/rebuilding several packages.
LF-05 may narrow it to exact checked per-package closures, but must use the same
schema for generation, receipts, and world validation. It must not substitute an
older package schema just because the source revision did not change.
Reusable package compilation now receives a canonical per-package build contract;
exact source provenance and workspace linking remain separate. See
[package build contracts](../../../docs/PACKAGE_BUILD_CONTRACTS.md). Generation,
receipts and checking must agree on that contract; unchanged source alone does
not justify reusing an incompatible binding schema.
## Transaction-port ABI
+43 -22
View File
@@ -2,7 +2,11 @@ import { graphExecutionRepresentation } from "./graph-execution.js";
import { lstat } from "node:fs/promises";
import { join } from "node:path";
import { hexDigest, ExecutionPlanError, type ExecutionPlan } from "./plan.js";
import { loadExecutionInstallation, type PackageInstallationSelection } from "./installation.js";
import {
loadExecutionInstallation,
createClientContractCatalog,
type PackageInstallationSelection,
} from "./installation.js";
import {
candidateMetadataBudget,
readCandidateMetadata,
@@ -36,33 +40,37 @@ export async function loadCheckedInstallationCandidate(
if (!rootPattern.test(root) || !(await lstat(root)).isDirectory())
throw Error("Candidate must be an immutable Nix output directory");
const scope = (resources.budget ?? candidateMetadataBudget).acquire(resources.owner ?? root);
let releaseValidation = () => {};
try {
// Validate the duplicated source contract by digest, then discard its tree
// before allocating the substantially larger checked execution plan.
const workspaceDigest = await (async () => {
const input = await readJson(root, "candidate.json", scope);
try {
const { executionContracts: _legacyContracts, ...workspace } = input.value;
return hexDigest(workspace);
} finally {
input.releaseTree();
}
})();
// Sequential reads avoid partially rejected Promise.all() work allocating
// after the failed candidate's reservation has been released.
const values = [];
for (const file of [
"candidate.json",
"execution-plan.json",
"execution-artifacts.json",
"source.json",
"entry.json",
"kernel.json",
"queries.json",
"graph-execution.json",
]) {
const parsed = await readJson(root, file, scope);
values.push(parsed.value);
// candidate.json duplicates the checked plan for validation. Nothing in
// the installed candidate retains this input tree after validation.
if (file === "candidate.json") releaseValidation = parsed.releaseTree;
}
const [candidate, plan, artifacts, source, entryRequest, kernel, queries, graph] = values;
const { executionContracts: _legacyContracts, ...workspace } = candidate;
const [plan, artifacts, source, entryRequest, kernel, queries] = values;
if (
plan?.definition?.codec !== "camino-execution-plan-v1" ||
hexDigest(plan.definition) !== plan.digest ||
hexDigest(workspace) !== hexDigest(plan.definition.workspace)
workspaceDigest !== hexDigest(plan.definition.workspace)
)
throw new ExecutionPlanError(
"CANDIDATE_PLAN_MISMATCH",
@@ -113,14 +121,24 @@ export async function loadCheckedInstallationCandidate(
"Stored interpreter input must match the checked build and plan",
);
const expectedGraph = graphExecutionRepresentation(plan, installation, kernel, queries);
if (hexDigest(graph) !== hexDigest(expectedGraph) || artifacts.execution !== hexDigest(graph))
throw new ExecutionPlanError(
"CANDIDATE_EXECUTION_MISMATCH",
root,
"Stored graph execution differs from checked inputs",
);
// The stored graph embeds the same kernel/query programs. Validate it in a
// short-lived scope and retain the representation that shares those trees.
await (async () => {
const input = await readJson(root, "graph-execution.json", scope);
try {
const digest = hexDigest(input.value);
if (digest !== hexDigest(expectedGraph) || artifacts.execution !== digest)
throw new ExecutionPlanError(
"CANDIDATE_EXECUTION_MISMATCH",
root,
"Stored graph execution differs from checked inputs",
);
} finally {
input.releaseTree();
}
})();
const provenance = {
workspaceRevision: workspace.id as string,
workspaceRevision: plan.definition.workspace.id as string,
source: { repository: source.repository as string, commit: source.commit as string },
generator: artifacts.generator as string,
packages: (plan as ExecutionPlan).definition.packages.map((pkg) => ({
@@ -139,30 +157,33 @@ export async function loadCheckedInstallationCandidate(
programs: artifacts.programs,
}),
};
const catalog = createClientContractCatalog(plan, installation, scope, kernel);
return {
reference,
workspace: plan.definition.workspace.workspaceId,
world: hexDigest(graph),
world: hexDigest(expectedGraph),
plan: kernel.plan as string,
graphExecution: graph,
graphExecution: expectedGraph,
provenance,
entry,
kernel,
queries,
installation,
executionPlan: plan as ExecutionPlan,
clientContractCatalog: catalog.select,
metadata: {
get bytes() {
return scope.bytes;
},
release: scope.release,
release: () => {
catalog.close();
scope.release();
},
},
};
} catch (error) {
scope.release();
throw error;
} finally {
releaseValidation();
}
}
+17 -14
View File
@@ -35,20 +35,23 @@ export interface InspectionTemplate {
}
/** A finite installed template per interface/lane, not a per-client query compiler.
* Flags select public cache boundaries; normal scope identity bounds specialization. */
export function compileInspectionTemplates(plan: ExecutionPlan) {
const kernel = compileKernelProgram(plan) as {
operations: {
key: string;
atom: string;
interface: string;
operation: string;
effect: string;
targets: string[];
unsupported: string | null;
output: unknown;
binding: { kind: string };
}[];
};
export interface InspectionKernel {
operations: {
key: string;
atom: string;
interface: string;
operation: string;
effect: string;
targets: string[];
unsupported: string | null;
output: unknown;
binding: { kind: string };
}[];
}
export function compileInspectionTemplates(
plan: ExecutionPlan,
kernel: InspectionKernel = compileKernelProgram(plan) as InspectionKernel,
) {
const queries: unknown[] = [],
contracts: PublicQueryContract[] = [],
templates: InspectionTemplate[] = [];
+72 -2
View File
@@ -1,4 +1,4 @@
import { compileInspectionTemplates, type InspectionTemplate } from "./inspection.js";
import { compileInspectionTemplates, type InspectionTemplate, type InspectionKernel } from "./inspection.js";
import path from "node:path";
import { loadCheckedArtifactWorld } from "./receipts.js";
import { contentArtifactUrl, loadArtifactWorld, type LoadedArtifactWorld } from "./files.js";
@@ -45,6 +45,7 @@ export function clientContractCatalog(
plan: ExecutionPlan,
installation: ExecutionInstallation,
requestedPackages: readonly string[],
checkedKernel?: InspectionKernel,
): ClientContractCatalog {
if (installation.plan !== plan.digest || hexDigest(plan.definition) !== plan.digest)
throw new ExecutionPlanError(
@@ -69,7 +70,7 @@ export function clientContractCatalog(
.map((operation) => publicOperationContract(iface, operation)),
),
);
const inspection = compileInspectionTemplates(plan);
const inspection = compileInspectionTemplates(plan, checkedKernel);
return {
inspections: inspection.templates,
resultInterfaces: Object.fromEntries(
@@ -98,6 +99,75 @@ export function clientContractCatalog(
};
}
/** Candidate-owned compiler output, never graph state or authorization. Compile
* once, then project each requested package set without hashing/recompiling the
* plan. Reserve bounded metadata before compilation; disposal releases both the
* catalog and its charge. Callers must supply the candidate's checked inputs. */
export function createClientContractCatalog(
plan: ExecutionPlan,
installation: ExecutionInstallation,
metadata: CandidateMetadataScope,
checkedKernel?: InspectionKernel,
) {
const revisions = plan.definition.workspace.packageImports.map((pkg) => pkg.revisionId);
const installed = new Set<string>(revisions);
let catalog: ClientContractCatalog | undefined;
let release: (() => void) | undefined;
let closed = false;
const freeze = (value: unknown, seen = new WeakSet<object>()) => {
if (!value || typeof value !== "object" || seen.has(value)) return;
seen.add(value);
for (const child of Object.values(value)) freeze(child, seen);
Object.freeze(value);
};
return {
select(requestedPackages: readonly string[]): ClientContractCatalog {
if (closed) throw Error("CANDIDATE_METADATA_CLOSED");
for (const revision of requestedPackages)
if (!installed.has(revision))
throw new ExecutionPlanError("UNINSTALLED_PACKAGE", revision, "Requested catalog package is not installed");
if (!catalog) {
// Like other candidate admission charges, this is an allocation estimate,
// not a V8 heap limit. The process/cgroup ceiling bounds transient work.
const reservation = metadata.reserve(32 * 1024 * 1024);
try {
const compiled = clientContractCatalog(plan, installation, revisions, checkedKernel);
const bytes = Buffer.byteLength(JSON.stringify(compiled));
if (bytes > 8 * 1024 * 1024) throw Error("CANDIDATE_CATALOG_LIMIT");
freeze(compiled);
// No await between reservations: replacing the transient upper bound
// with a smaller retained estimate cannot race another admission.
reservation();
release = metadata.reserve(Math.max(64 * 1024, bytes * 4));
catalog = compiled;
} catch (error) {
reservation();
throw error;
}
}
const requested = new Set(requestedPackages);
return {
inspections: catalog.inspections,
resultInterfaces: catalog.resultInterfaces,
operations: catalog.operations,
queries: [
...catalog.queries.filter((query) => !installed.has(query.package)),
...requestedPackages.flatMap((revision) => catalog!.queries.filter((query) => query.package === revision)),
],
components: catalog.components.filter((component) => requested.has(component.packageRevision)),
moduleBindings: catalog.moduleBindings.filter((binding) => requested.has(binding.packageRevision)),
presentations: catalog.presentations.filter((presentation) => requested.has(presentation.packageRevision)),
};
},
close() {
closed = true;
catalog = undefined;
release?.();
release = undefined;
},
};
}
export interface PackageInstallationSelection {
readonly browser?: { readonly id?: string };
readonly service?: { readonly id?: string };
+5 -1
View File
@@ -9,7 +9,11 @@ export class CandidateMetadataBudget {
private bytes = 0;
private scopes = 0;
private readonly owners = new Map<string, number>();
constructor(limits = { bytes: 512 * 1024 * 1024, ownerBytes: 256 * 1024 * 1024, candidates: 128 }) {
// A cutover holds both the serving and proposed installation plus transient
// parsing space. The 57-repository reader retains ~133 MiB per installation;
// a 256 MiB owner ceiling cannot admit its ordinary atomic replacement.
// Keep the aggregate ceiling independent and leave room for parsing overlap.
constructor(limits = { bytes: 512 * 1024 * 1024, ownerBytes: 384 * 1024 * 1024, candidates: 128 }) {
for (const value of Object.values(limits))
if (!Number.isSafeInteger(value) || value < 1) throw Error("INVALID_CANDIDATE_METADATA_BUDGET");
if (limits.ownerBytes > limits.bytes) throw Error("INVALID_CANDIDATE_METADATA_BUDGET");
+39 -3
View File
@@ -26,13 +26,41 @@ export interface ImportFinding {
export const restrictedSDKSpecifier = "quixos:restricted-sdk/v1";
export const restrictedSDKPath = "share/quixos/restricted/core.mjs";
/** Compare trusted SDK syntax without build-machine paths in source comments.
* Keep literal/directive spelling and every executable AST node. This is SDK
* compatibility only: artifact files and the executed host SDK still have exact
* byte digests, and authored SDK bytes are never loaded by the restricted host.
*/
export function restrictedSDKIdentity(source: string): string {
if (Buffer.byteLength(source) > 2 * 1024 * 1024) throw Error("RESTRICTED_SDK_ANALYSIS_LIMIT");
const syntax = parse(source, { sourceType: "module", attachComment: false });
const metadata = new Set([
"start",
"end",
"loc",
"comments",
"leadingComments",
"innerComments",
"trailingComments",
"parenStart",
]);
let nodes = 0;
const encoded = JSON.stringify(syntax, (key, value) => {
if (metadata.has(key)) return undefined;
if (value && typeof value === "object" && typeof value.type === "string" && ++nodes > 100000)
throw Error("RESTRICTED_SDK_ANALYSIS_LIMIT");
return value;
});
return hexDigest({ codec: "quixos-restricted-sdk-syntax-v1", syntax: JSON.parse(encoded) });
}
/** Mechanical candidate checks, not authorization or an engine-safety proof.
* The caller must first verify the manifest and all declared file digests. */
export async function inspectRestrictedImplementation(
revision: string,
implementation: ArtifactImplementation,
read: ArtifactReader,
qualifiedSDKDigest?: string,
qualifiedSDKIdentity?: string,
): Promise<ImportFinding[]> {
const entry = implementation.files.find((file) => file.path === implementation.path);
const sdk = implementation.files.find((file) => file.path === restrictedSDKPath);
@@ -48,10 +76,18 @@ export async function inspectRestrictedImplementation(
!/^[$A-Z_a-z][$\w]*$/.test(implementation.registryExport ?? "")
)
return [{ code: "RESTRICTED_CLOSURE", detail: "Expected one closed module and the restricted SDK", line: null }];
if (qualifiedSDKDigest && sdk.digest !== qualifiedSDKDigest)
return [{ code: "RESTRICTED_SDK_UNSUPPORTED", detail: "SDK differs from the qualified host profile", line: null }];
if (entry.bytes > 1024 * 1024 || sdk.bytes > 2 * 1024 * 1024)
return [{ code: "ANALYSIS_LIMIT", detail: "Restricted closure exceeds the source budget", line: null }];
if (qualifiedSDKIdentity) {
const source = Buffer.from(await read(revision, sdk.path, sdk.bytes)).toString("utf8");
try {
if (restrictedSDKIdentity(source) !== qualifiedSDKIdentity) throw Error("SDK syntax differs");
} catch {
return [
{ code: "RESTRICTED_SDK_UNSUPPORTED", detail: "SDK differs from the qualified host profile", line: null },
];
}
}
return inspectJavaScriptSurface(Buffer.from(await read(revision, entry.path, entry.bytes)).toString("utf8"), [
restrictedSDKSpecifier,
]);
+56
View File
@@ -4,6 +4,62 @@ import { mkdtemp, writeFile, symlink, rm } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { CandidateMetadataBudget, readCandidateMetadata } from "../src/execution-world/metadata.js";
import { loadCheckedInstallationCandidate } from "../src/execution-world/candidate.js";
test(
"a real checked application can stage its replacement within the default metadata budget",
{
skip: !process.env.QX_CHECKED_CANDIDATE,
},
async () => {
const root = process.env.QX_CHECKED_CANDIDATE!;
const budget = new CandidateMetadataBudget();
const first = await loadCheckedInstallationCandidate(root, {}, { budget, owner: "reader" });
try {
const retained = budget.snapshot().bytes;
const second = await loadCheckedInstallationCandidate(root, {}, { budget, owner: "reader" });
try {
assert.equal(budget.snapshot().bytes, retained * 2);
assert.deepEqual(second.reference, first.reference, "Compaction preserves exact checked identity");
assert.equal(second.graphExecution.kernel, second.kernel, "Graph representation shares the validated program");
assert.equal(second.graphExecution.queries, second.queries);
// A deliberately tighter pool admits the serving installation but must
// reject replacement parsing without leaking a candidate or tree lease.
const tight = new CandidateMetadataBudget({
bytes: retained + 1,
ownerBytes: retained + 1,
candidates: 2,
});
const blocker = tight.acquire("reader");
blocker.charge(retained);
try {
await assert.rejects(
loadCheckedInstallationCandidate(root, {}, { budget: tight, owner: "reader" }),
/CANDIDATE_METADATA_LIMIT/,
);
assert.equal(tight.snapshot().bytes, retained);
assert.equal(tight.snapshot().candidates, 1);
} finally {
blocker.release();
}
const other = budget.acquire("other");
try {
assert.throws(() => other.charge(budget.limits.bytes), /CANDIDATE_METADATA_LIMIT/);
assert.equal(budget.snapshot().bytes, retained * 2, "Rejected admission must not leak charges");
} finally {
other.release();
}
} finally {
second.metadata.release();
}
assert.equal(budget.snapshot().bytes, retained, "The serving candidate survives failed or completed staging");
} finally {
first.metadata.release();
}
assert.equal(budget.snapshot().bytes, 0);
assert.equal(budget.snapshot().candidates, 0);
},
);
test("concurrent candidate reservations enforce aggregate and owner limits and release exactly once", () => {
const budget = new CandidateMetadataBudget({ bytes: 1000, ownerBytes: 600, candidates: 3 });
+54
View File
@@ -6,10 +6,64 @@ import {
compileExecutionPlan,
hexDigest,
inspectJavaScriptSurface,
inspectRestrictedImplementation,
restrictedSDKIdentity,
restrictedSDKPath,
type ArtifactImplementation,
reportExecutionPlacement,
type ExecutionInstallation,
} from "../src/execution-world/index.js";
test("restricted SDK identity ignores build comments and positions, preserving executable syntax", () => {
const source = 'export function value() { return ("hello"); }';
const x86 = "// /nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-core/dist/portable.js\n" + source;
const arm = "// /nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-core/dist/portable.js\n\n" + source;
assert.equal(restrictedSDKIdentity(x86), restrictedSDKIdentity(arm));
for (const changed of [
'export function value() { return ("goodbye"); }',
'export function value() { return\n("hello"); }',
'export function value() { fetch("hello"); }',
'import x from "node:fs"; ' + source,
'export function value() { return "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-core"; }',
])
assert.notEqual(restrictedSDKIdentity(source), restrictedSDKIdentity(changed));
assert.notEqual(restrictedSDKIdentity('"use strict";'), restrictedSDKIdentity('"use\\x20strict";'));
assert.notEqual(restrictedSDKIdentity('const value="// comment";'), restrictedSDKIdentity('const value="";'));
assert.throws(() => restrictedSDKIdentity("const ???"));
assert.throws(() => restrictedSDKIdentity(" ".repeat(2 * 1024 * 1024 + 1)), /ANALYSIS_LIMIT/);
});
test("restricted admission accepts the same SDK syntax and rejects changed SDK behavior", async () => {
const source = "export const registry = {};";
const sdk = 'export const value = "trusted";';
const implementation: ArtifactImplementation = {
id: "restricted",
target: "server",
compatibility: "quixos-restricted-module-v1",
path: "module.mjs",
ports: "quixos-transaction-ports-v1",
registryExport: "registry",
exports: ["value"],
contentDigest: "a".repeat(64),
files: [
{ path: "module.mjs", digest: "b".repeat(64), bytes: source.length, mediaType: "text/javascript" },
{ path: restrictedSDKPath, digest: "c".repeat(64), bytes: sdk.length, mediaType: "text/javascript" },
],
};
const check = (candidateSDK: string) => {
implementation.files[1]!.bytes = Buffer.byteLength(candidateSDK);
return inspectRestrictedImplementation(
"package:test@1",
implementation,
async (_revision, path) => Buffer.from(path === restrictedSDKPath ? candidateSDK : source),
restrictedSDKIdentity(sdk),
);
};
assert.deepEqual(await check("// a different build path\n" + sdk), []);
assert.equal((await check('export const value = "untrusted";'))[0]?.code, "RESTRICTED_SDK_UNSUPPORTED");
assert.equal((await check("export const ???"))[0]?.code, "RESTRICTED_SDK_UNSUPPORTED");
});
test("compiled-source inspection finds authority imports and ambient facilities without executing code", () => {
const surface = inspectJavaScriptSurface(
`
+54
View File
@@ -19,12 +19,16 @@ import {
expectedArtifactReceipt,
checkExecutionInstallation,
clientContractCatalog,
createClientContractCatalog,
CandidateMetadataBudget,
compileKernelProgram,
type ComponentArtifact,
type PackageArtifacts,
} from "../src/execution-world/index.js";
import { validateWorkspaceRevision } from "../src/capability-model/index.js";
import { presentationDeclarations, preparationArtifact } from "../src/react-build/declarations.js";
import { compileQueryComposition } from "../src/query/composition.js";
import type { InspectionKernel } from "../src/execution-world/inspection.js";
const source = { repository: "https://example.test/components.git", commit: "a".repeat(40) };
const contract = (name: string) => {
@@ -255,6 +259,56 @@ test("checked component-only artifacts install without a service and reject miss
() => clientContractCatalog(plan, { ...installation, plan: "bad" }, [pkg.revision]),
/CATALOG_PLAN_MISMATCH/,
);
const budget = new CandidateMetadataBudget();
const scope = budget.acquire("catalog-test");
const prepared = createClientContractCatalog(
plan,
installation,
scope,
compileKernelProgram(plan) as InspectionKernel,
);
try {
assert.throws(() => prepared.select(["absent"]), /UNINSTALLED_PACKAGE/);
assert.equal(scope.bytes, 0, "Unknown requests do not allocate compiler metadata");
assert.deepEqual(prepared.select([pkg.revision]), catalog);
const retained = scope.bytes;
assert.ok(retained > 0);
assert.ok(retained < 32 * 1024 * 1024, "Small catalogs retain a proportional metadata charge");
const full = prepared.select([pkg.revision]);
const empty = prepared.select([]);
assert.deepEqual(empty.components, []);
assert.deepEqual(empty.presentations, []);
assert.deepEqual(empty.moduleBindings, []);
assert.equal(empty.operations, full.operations, "Shared compiled metadata is reused");
assert.equal(Object.isFrozen(full.operations), true);
// Any plan access now would reveal a repeated digest/compile on the warm path.
const definition = plan.definition;
Object.defineProperty(plan, "definition", {
configurable: true,
get() {
throw Error("RECOMPILED_PLAN");
},
});
try {
assert.deepEqual(prepared.select([pkg.revision]), full);
assert.equal(scope.bytes, retained, "Package projections do not accumulate cache entries");
} finally {
Object.defineProperty(plan, "definition", { configurable: true, value: definition });
}
prepared.close();
assert.equal(scope.bytes, 0);
assert.throws(() => prepared.select([pkg.revision]), /CANDIDATE_METADATA_CLOSED/);
} finally {
prepared.close();
scope.release();
}
const tight = new CandidateMetadataBudget({ bytes: 1000, ownerBytes: 1000, candidates: 1 });
const tightScope = tight.acquire("catalog-test");
const refused = createClientContractCatalog(plan, installation, tightScope);
assert.throws(() => refused.select([pkg.revision]), /CANDIDATE_METADATA_LIMIT/);
assert.equal(tightScope.bytes, 0);
refused.close();
tightScope.release();
} finally {
await rm(root, { recursive: true, force: true });
}