Export template source from 11bc3ae35b0034c113daca4c35d1b341aa31ef70:examples/local-first/packages/language-reader-runtime
This commit is contained in:
Quixos Template Publisher
@@ -0,0 +1,42 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { limitedBody, synthesize, wikipediaClient } from "../src/network.js";
|
||||
const signal = new AbortController().signal;
|
||||
let calls: { url: string; init?: RequestInit }[] = [];
|
||||
const mock = (async (url: URL | RequestInfo, init?: RequestInit) => {
|
||||
calls.push({ url: String(url), init });
|
||||
if (String(url).includes("list=search")) return Response.json({ query: { search: [{ title: "北京" }] } });
|
||||
return Response.json({ query: { pages: [{ title: "北京", extract: "北京是中国的首都。" }] } });
|
||||
}) as typeof fetch;
|
||||
const wiki = wikipediaClient(mock);
|
||||
assert.deepEqual(await wiki.search("北京", signal), [{ title: "北京", description: "Chinese Wikipedia" }]);
|
||||
const page = await wiki.read("北京", signal);
|
||||
assert.equal(page.text, "北京是中国的首都。");
|
||||
assert(page.attribution.includes("CC BY-SA"));
|
||||
assert(page.url.startsWith("https://zh.wikipedia.org/wiki/"));
|
||||
const count = calls.length;
|
||||
await wiki.read("北京", signal);
|
||||
assert.equal(calls.length, count, "Repeated loads use the bounded cache");
|
||||
assert((calls[0].init?.headers as Record<string, string>)["user-agent"].includes("QuixosReading"));
|
||||
assert(new URL(calls[0].url).searchParams.has("maxlag"));
|
||||
await assert.rejects(() => wiki.read("a|b", signal), /Invalid/);
|
||||
await assert.rejects(() => limitedBody(new Response("x".repeat(50)), 10), /too large/);
|
||||
await assert.rejects(
|
||||
() => limitedBody(new Response("", { status: 429, headers: { "retry-after": "10" } }), 10),
|
||||
/Retry after 10/,
|
||||
);
|
||||
let speechRequest: RequestInit | undefined;
|
||||
const speech = (async (url: URL | RequestInfo, init?: RequestInit) => {
|
||||
assert(String(url).startsWith("https://api.elevenlabs.io/v1/text-to-speech/voice/stream"));
|
||||
speechRequest = init;
|
||||
return new Response(new Uint8Array([1, 2, 3]), { headers: { "content-type": "audio/mpeg" } });
|
||||
}) as typeof fetch;
|
||||
assert.deepEqual(await synthesize("你好", "fixture-private-key", "voice", signal, speech), new Uint8Array([1, 2, 3]));
|
||||
assert.equal((speechRequest?.headers as Record<string, string>)["xi-api-key"], "fixture-private-key");
|
||||
assert.equal(JSON.parse(speechRequest?.body as string).model_id, "eleven_multilingual_v2");
|
||||
await assert.rejects(() => synthesize("你好", "", "voice", signal, speech), /API key/);
|
||||
await assert.rejects(() => synthesize("中".repeat(1201), "key", "voice", signal, speech), /1,200/);
|
||||
await assert.rejects(
|
||||
() => synthesize("你好", "key", "voice", signal, (async () => new Response("", { status: 401 })) as typeof fetch),
|
||||
/did not accept/,
|
||||
);
|
||||
console.log("PASS Wikipedia bounds, identification, attribution, cache and speech headers/errors");
|
||||
@@ -0,0 +1,357 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { processors, validateAnnotations, AnnotationValidationError, annotationSchemaFor } from "../src/processors.js";
|
||||
const source = "北京👩🏽💻欢迎你";
|
||||
const item = { start: 0, end: 2, source: "北京", text: "Běijīng", tag: "place" };
|
||||
const renderedItem = { start: 0, end: 2, text: "Běijīng", tag: "place" };
|
||||
assert.deepEqual(validateAnnotations({ items: [item] }, source, 0, source.length), [renderedItem]);
|
||||
for (const value of [
|
||||
null,
|
||||
{ items: [null] },
|
||||
{ items: [{ ...item, end: 999 }] },
|
||||
{ items: [{ ...item, start: 1, end: 0 }] },
|
||||
{ items: [{ ...item, extra: true }] },
|
||||
{ items: [{ ...item, text: 8 }] },
|
||||
{ items: [{ ...item, start: 3, end: 4 }] },
|
||||
])
|
||||
assert.throws(() => validateAnnotations(value, source, 0, source.length));
|
||||
assert.throws(() => validateAnnotations({ items: Array(181).fill(item) }, source, 0, source.length));
|
||||
assert.throws(() => validateAnnotations({ items: [item] }, source, 2, source.length));
|
||||
assert.equal(processors.definition.strategy, "focus");
|
||||
assert.equal(processors.grammar.strategy, "section");
|
||||
assert.deepEqual(processors.punctuation.local!("你好。", 4), [
|
||||
{ start: 6, end: 7, text: "Sentence boundary", tag: "punctuation" },
|
||||
]);
|
||||
console.log("PASS structured annotation validation, scope limits and processor strategies");
|
||||
|
||||
import { normalizeConfig, fieldsFor } from "../src/config.js";
|
||||
const defaultOptions = normalizeConfig(processors.grammar, "{}");
|
||||
assert.equal(defaultOptions.nounColor, "#6a8aac");
|
||||
assert.equal(fieldsFor(processors.grammar).length, 6);
|
||||
const customized = normalizeConfig(
|
||||
processors.grammar,
|
||||
JSON.stringify({ nounColor: "#112233", fontFamily: "monospace", fontSize: 24 }),
|
||||
);
|
||||
assert.equal(customized.fontSize, 24);
|
||||
assert.equal(defaultOptions.underlinePattern, "solid");
|
||||
assert.equal(defaultOptions.placement, "above");
|
||||
assert.equal(normalizeConfig(processors.pinyin, '{"placement":"below"}').placement, "below");
|
||||
for (const placement of ["under", "", null, 1])
|
||||
assert.throws(() => normalizeConfig(processors.pinyin, JSON.stringify({ placement })), /Label position/);
|
||||
for (const underlinePattern of ["solid", "dashed", "squiggly"]) {
|
||||
assert.equal(
|
||||
normalizeConfig(processors.grammar, JSON.stringify({ underlinePattern })).underlinePattern,
|
||||
underlinePattern,
|
||||
);
|
||||
}
|
||||
for (const underlinePattern of ["wavy", "", null, 1]) {
|
||||
assert.throws(() => normalizeConfig(processors.grammar, JSON.stringify({ underlinePattern })), /Underline pattern/);
|
||||
}
|
||||
for (const options of [
|
||||
"[]",
|
||||
"null",
|
||||
'{"unknown":true}',
|
||||
'{"fontSize":0}',
|
||||
'{"fontSize":null}',
|
||||
'{"fontSize":80}',
|
||||
'{"fontSize":"24"}',
|
||||
'{"nounColor":"red"}',
|
||||
'{"minimumLevel":"not-a-level"}',
|
||||
'{"fontFamily":"bad\\nfont"}',
|
||||
]) {
|
||||
assert.throws(() =>
|
||||
normalizeConfig(options.includes("minimumLevel") ? processors.pinyin : processors.grammar, options),
|
||||
);
|
||||
}
|
||||
assert.equal(processors.pinyin.prompt, undefined);
|
||||
console.log("PASS generated annotator config, validation, prompt options and dynamic typography/color");
|
||||
|
||||
import { TextIndex, processingPolicy, textIndex } from "../src/text-processing.js";
|
||||
const policy = processingPolicy();
|
||||
const article = Array.from({ length: 300 }, (_, n) => `第${n}句，阅读👩🏽💻和e\u0301。下一句提供上下文。\n`).join("");
|
||||
const index = new TextIndex(article),
|
||||
chunks = index.chunks(policy.chunkChars);
|
||||
assert.equal(chunks.map((range) => article.slice(range.start, range.end)).join(""), article);
|
||||
assert.equal(chunks[0].start, 0);
|
||||
assert.equal(chunks.at(-1)!.end, article.length);
|
||||
for (const range of chunks) {
|
||||
assert.deepEqual(index.range(range), range);
|
||||
assert(new TextIndex(article.slice(range.start, range.end)).units.length <= policy.chunkChars);
|
||||
}
|
||||
for (let start = 0; start < article.length; start += 57) {
|
||||
const window = { start, end: Math.min(article.length, start + 500) };
|
||||
const nearby = index.nearby(window, { start, end: start + 100 }, policy);
|
||||
assert(nearby.length <= 3);
|
||||
assert(
|
||||
nearby.every((chunk) => chunks.includes(chunk)),
|
||||
"Overlapping viewports reuse stable chunk identities",
|
||||
);
|
||||
assert(nearby.every((chunk) => chunk.end > window.start && chunk.start < window.end));
|
||||
const context = index.context(window, policy);
|
||||
assert(new TextIndex(article.slice(context.start, context.end)).units.length <= policy.contextChars);
|
||||
}
|
||||
const sentences = new TextIndex(
|
||||
"First sentence. Second sentence. Third sentence. Fourth sentence. Fifth sentence. Sixth sentence.",
|
||||
);
|
||||
const fourth = sentences.sentences[3];
|
||||
assert.deepEqual(sentences.focus({ start: fourth.start + 4, end: fourth.start + 5 }, "sentence", policy), fourth);
|
||||
assert.equal(sentences.context(fourth, policy).start, sentences.sentences[1].start);
|
||||
assert.equal(sentences.context(fourth, policy).end, sentences.text.length);
|
||||
const huge = new TextIndex("无".repeat(10000));
|
||||
for (const scope of ["word", "sentence", "section"]) {
|
||||
const focused = huge.focus({ start: 5000, end: 9900 }, scope, policy);
|
||||
assert(focused.end - focused.start <= 400, "No document-sized click, even without punctuation");
|
||||
assert(focused.start <= 5000 && focused.end > 5000);
|
||||
assert(huge.context(focused, policy).end - huge.context(focused, policy).start <= 1600);
|
||||
}
|
||||
assert.equal(
|
||||
huge
|
||||
.chunks(800)
|
||||
.map((range) => huge.text.slice(range.start, range.end))
|
||||
.join(""),
|
||||
huge.text,
|
||||
);
|
||||
const emoji = new TextIndex("👩🏽💻e\u0301汉字");
|
||||
assert.deepEqual(emoji.range({ start: 1, end: 2 }), { start: 0, end: 7 });
|
||||
assert.deepEqual(emoji.tokens({ start: 7, end: 9 }), [{ start: 7, end: 9, text: "e\u0301" }]);
|
||||
const longRange = processingPolicy({
|
||||
context: "document",
|
||||
contextChars: 80000,
|
||||
focusChars: 2000,
|
||||
chunkChars: 3000,
|
||||
prefetchChunks: 2,
|
||||
});
|
||||
assert.deepEqual(index.context({ start: 1000, end: 1001 }, longRange), { start: 0, end: article.length });
|
||||
assert(
|
||||
huge.focus({ start: 5000, end: 5001 }, "sentence", longRange).end -
|
||||
huge.focus({ start: 5000, end: 5001 }, "sentence", longRange).start >
|
||||
400,
|
||||
);
|
||||
assert(index.nearby({ start: 3000, end: 3100 }, { start: 3000, end: 3100 }, longRange).length > 1);
|
||||
for (const overrides of [
|
||||
{ chunkChars: 0 },
|
||||
{ focusChars: 80001 },
|
||||
{ contextChars: NaN },
|
||||
{ beforeSentences: -1 },
|
||||
{ prefetchChunks: 21 },
|
||||
])
|
||||
assert.throws(() => processingPolicy(overrides));
|
||||
assert.equal(textIndex(article), textIndex(article));
|
||||
assert.equal(processors.passageTranslation.strategy, "section");
|
||||
console.log(
|
||||
"PASS reusable text planner: multilingual source offsets, stable bounded chunks, sentence context, pathological text, author overrides",
|
||||
);
|
||||
|
||||
import { createVariant, nextSection, applySection, variantText } from "../src/variants.js";
|
||||
const largeSource = "这是一个句子。\n".repeat(3000) + "👩🏽💻结束。";
|
||||
let variant = createVariant(largeSource, "source-revision", "english");
|
||||
assert(variant.sections.length > 10);
|
||||
assert.equal(variant.sections.map((s) => largeSource.slice(s.start, s.end)).join(""), largeSource);
|
||||
assert(
|
||||
variant.sections.every(
|
||||
(s) =>
|
||||
Array.from(new Intl.Segmenter(undefined, { granularity: "grapheme" }).segment(largeSource.slice(s.start, s.end)))
|
||||
.length <= 1400,
|
||||
),
|
||||
);
|
||||
const first = nextSection(variant)!;
|
||||
assert.throws(() => applySection(variant, first.start + 1, first.end, "bad"));
|
||||
variant = applySection(variant, first.start, first.end, "A shorter section.");
|
||||
const second = nextSection(variant)!;
|
||||
variant = applySection(
|
||||
variant,
|
||||
second.start,
|
||||
second.end,
|
||||
"A much longer translated second section, with different output offsets.",
|
||||
);
|
||||
assert.equal(
|
||||
variantText(variant),
|
||||
"A shorter section.\n\nA much longer translated second section, with different output offsets.",
|
||||
);
|
||||
assert.equal(variant.sections[1].start, second.start, "Canonical section identities survive length changes");
|
||||
console.log(
|
||||
"PASS lazy variants: bounded Unicode sections, exact canonical coverage, sequential append and changing output lengths",
|
||||
);
|
||||
|
||||
import { computationFields } from "../src/config.js";
|
||||
import { cacheConfiguration, findCachedJob, sameCacheConfiguration } from "../src/cache.js";
|
||||
const cached = (processor: typeof processors.pinyin, options: Record<string, unknown>) =>
|
||||
cacheConfiguration("pinyin", "word", "section", options, computationFields(processor));
|
||||
assert.equal(
|
||||
cached(processors.pinyin, { minimumLevel: "2", fontSize: 13 }),
|
||||
cached(processors.pinyin, { minimumLevel: "8", fontSize: 32 }),
|
||||
"HSK/font display changes do not bust computation",
|
||||
);
|
||||
assert.deepEqual(computationFields(processors.names), [], "Per-category colors never reach inference");
|
||||
assert.equal(
|
||||
cached(processors.grammar, normalizeConfig(processors.grammar, '{"underlinePattern":"solid"}')),
|
||||
cached(processors.grammar, normalizeConfig(processors.grammar, '{"underlinePattern":"squiggly"}')),
|
||||
"Changing underline geometry reuses the same computed annotation results",
|
||||
);
|
||||
assert.equal(
|
||||
cached(processors.pinyin, normalizeConfig(processors.pinyin, '{"underlinePattern":"dashed"}')),
|
||||
cached(processors.pinyin, normalizeConfig(processors.pinyin, "{}")),
|
||||
"Common underline appearance never reaches semantic prompt configuration",
|
||||
);
|
||||
const custom = { ...processors.pinyin, cache: { inputs: ["targetLanguage"] } };
|
||||
assert.notEqual(
|
||||
cached(custom, { targetLanguage: "English" }),
|
||||
cached(custom, { targetLanguage: "French" }),
|
||||
"Semantic inputs isolate caches",
|
||||
);
|
||||
assert.equal(
|
||||
cacheConfiguration("pinyin", "word", "focus", {}, []),
|
||||
cached(processors.pinyin, {}),
|
||||
"Generation timing is independent of semantic results",
|
||||
);
|
||||
console.log("PASS semantic cache configuration, display-only HSK/color/font edits and explicit author inputs");
|
||||
|
||||
let activeReads = 0,
|
||||
maximumReads = 0;
|
||||
const history = Array.from({ length: 97 }, (_, index) => index);
|
||||
const read = async (index: number) => {
|
||||
maximumReads = Math.max(maximumReads, ++activeReads);
|
||||
await Promise.resolve();
|
||||
activeReads--;
|
||||
return { key: index === 34 || index === 79 ? "wanted" : "other", status: index === 34 ? "failed" : "ready" };
|
||||
};
|
||||
assert.deepEqual(await findCachedJob(history, read, "wanted", true), { reference: undefined, retries: 2 });
|
||||
assert(maximumReads > 1 && maximumReads <= 32, "Historical cache reads run concurrently with a bound");
|
||||
assert.deepEqual(await findCachedJob(history, read, "wanted", false), { reference: 79, retries: 2 });
|
||||
assert.deepEqual(await findCachedJob(history, read, "missing", true), { reference: undefined, retries: 0 });
|
||||
console.log("PASS bounded cache admission, stable reuse order and retry accounting");
|
||||
|
||||
import { processorIdentity } from "../src/processor-identity.js";
|
||||
const identity = (processor: typeof processors.pinyin, source = "package@1", task = "task@1") =>
|
||||
processorIdentity("pinyin", processor, {}, source, task);
|
||||
assert.deepEqual(
|
||||
identity(processors.pinyin),
|
||||
identity(processors.pinyin, "package@2"),
|
||||
"Unrelated runtime changes retain declarative model results",
|
||||
);
|
||||
assert.deepEqual(
|
||||
identity(processors.pinyin),
|
||||
identity({ ...processors.pinyin, renders: [{ style: "tooltip", reveal: "hover" }] }),
|
||||
"Render hooks do not invalidate inference",
|
||||
);
|
||||
assert.notDeepEqual(identity(processors.pinyin), identity({ ...processors.pinyin, instructions: "Different task" }));
|
||||
assert.notDeepEqual(
|
||||
identity(processors.pinyin),
|
||||
identity({ ...processors.pinyin, processing: { contextChars: 800 } }),
|
||||
"Context policy changes invalidate previously rendered results",
|
||||
);
|
||||
assert.deepEqual(
|
||||
identity({ ...processors.pinyin, processing: { contextChars: 800, chunkChars: 500, prefetchChunks: 1 } }),
|
||||
identity({ ...processors.pinyin, processing: { contextChars: 800, chunkChars: 900, prefetchChunks: 3 } }),
|
||||
"Chunking and prefetch preferences do not invalidate semantic results",
|
||||
);
|
||||
assert.deepEqual(
|
||||
identity(processors.pinyin),
|
||||
identity({ ...processors.pinyin, profile: "cerebras-light" }),
|
||||
"Routing preferences do not discard acceptable semantic results",
|
||||
);
|
||||
const configuredProcessor = {
|
||||
...processors.pinyin,
|
||||
config: [{ key: "targetLanguage", label: "Language", kind: "text" as const, default: "English" }],
|
||||
};
|
||||
assert.deepEqual(
|
||||
identity(configuredProcessor),
|
||||
identity({
|
||||
...configuredProcessor,
|
||||
config: [{ ...configuredProcessor.config[0], label: "Translate into", default: "French" }],
|
||||
}),
|
||||
"Config labels and defaults do not alter an already chosen semantic value",
|
||||
);
|
||||
assert.deepEqual(
|
||||
identity(processors.pinyin),
|
||||
identity(processors.pinyin, "package@1", "task@2"),
|
||||
"Actual tokens/context key the result; unrelated planner edits do not erase semantic identity",
|
||||
);
|
||||
const localProcessor = { ...processors.pinyin, local: () => [] };
|
||||
assert.notDeepEqual(
|
||||
identity(localProcessor),
|
||||
identity(localProcessor, "package@2"),
|
||||
"Imported local helpers remain part of identity",
|
||||
);
|
||||
const versioned = { ...localProcessor, cache: { version: "semantic@1" } };
|
||||
assert.deepEqual(identity(versioned), identity(versioned, "package@2"));
|
||||
assert.notDeepEqual(identity(versioned), identity({ ...versioned, cache: { version: "semantic@2" } }));
|
||||
assert.notDeepEqual(
|
||||
identity(versioned),
|
||||
identity({ ...versioned, local: () => [{ start: 0, end: 1, text: "changed", tag: "" }] }),
|
||||
);
|
||||
assert(
|
||||
sameCacheConfiguration(
|
||||
cached(processors.pinyin, {}),
|
||||
JSON.stringify({ processor: "pinyin", scope: "word", generation: "focus", options: {} }),
|
||||
),
|
||||
);
|
||||
assert(
|
||||
!sameCacheConfiguration(cached(custom, { targetLanguage: "French" }), cached(custom, { targetLanguage: "English" })),
|
||||
);
|
||||
assert(!sameCacheConfiguration("malformed", "also malformed"));
|
||||
console.log(
|
||||
"PASS semantic processor revisions, routing-independent results, explicit code versions and legacy scheduling normalization",
|
||||
);
|
||||
|
||||
const mismatched = [
|
||||
{ ...item, source: "欢迎" },
|
||||
{ ...item, start: 2, end: 9, source: "wrong" },
|
||||
];
|
||||
try {
|
||||
validateAnnotations({ items: mismatched }, source, 0, source.length);
|
||||
assert.fail("Expected source mismatches");
|
||||
} catch (error) {
|
||||
assert(error instanceof AnnotationValidationError);
|
||||
assert.equal(error.errors.length, 2);
|
||||
assert(error.errors[0].includes("items[0].source"));
|
||||
assert(error.errors[1].includes("items[1].source"));
|
||||
}
|
||||
assert.throws(() => validateAnnotations({ items: [renderedItem] }, source, 0, source.length), /source/);
|
||||
assert.deepEqual(validateAnnotations({ items: [renderedItem] }, source, 0, source.length, false), [renderedItem]);
|
||||
assert(annotationSchemaFor(processors.pinyin).properties.items.items.required.includes("source"));
|
||||
assert(
|
||||
!annotationSchemaFor({ ...processors.pinyin, verifySource: false }).properties.items.items.required.includes(
|
||||
"source",
|
||||
),
|
||||
);
|
||||
const repeated = "café café 👩🏽💻 e\u0301";
|
||||
assert.deepEqual(
|
||||
validateAnnotations({ items: [{ start: 5, end: 9, source: "café", text: "coffee", tag: "noun" }] }, repeated, 5, 9),
|
||||
[{ start: 5, end: 9, text: "coffee", tag: "noun" }],
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
validateAnnotations(
|
||||
{ items: [{ start: 10, end: 17, source: "👩", text: "developer", tag: "noun" }] },
|
||||
repeated,
|
||||
0,
|
||||
repeated.length,
|
||||
),
|
||||
/source/,
|
||||
);
|
||||
console.log("PASS exact source echo, all indexed mismatches, repeated words, UTF-16 and explicit code-owned opt-out");
|
||||
|
||||
try {
|
||||
validateAnnotations(
|
||||
{
|
||||
items: Array.from({ length: 180 }, () => ({
|
||||
start: 0,
|
||||
end: 120,
|
||||
source: "\u0002".repeat(120),
|
||||
text: "meaning",
|
||||
tag: "",
|
||||
})),
|
||||
},
|
||||
"\u0001".repeat(120),
|
||||
0,
|
||||
120,
|
||||
);
|
||||
assert.fail("Expected all mismatches");
|
||||
} catch (error) {
|
||||
assert(error instanceof AnnotationValidationError);
|
||||
assert.equal(error.errors.length, 180);
|
||||
assert(
|
||||
new TextEncoder().encode(JSON.stringify(error.errors)).length < 49152,
|
||||
"Every located error fits in one bounded feedback turn",
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user