376 lines
17 KiB
TypeScript
376 lines
17 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { processors, validateAnnotations, AnnotationValidationError, annotationSchemaFor } from "../src/processors.js";
|
|
const source = "北京👩🏽💻欢迎你";
|
|
const item = { start: 0, end: 2, source: "北京", text: "Běijīng", tag: "place" };
|
|
const renderedItem = { start: 0, end: 2, text: "Běijīng", tag: "place" };
|
|
assert.deepEqual(validateAnnotations({ items: [item] }, source, 0, source.length), [renderedItem]);
|
|
for (const value of [
|
|
null,
|
|
{ items: [null] },
|
|
{ items: [{ ...item, end: 999 }] },
|
|
{ items: [{ ...item, start: 1, end: 0 }] },
|
|
{ items: [{ ...item, extra: true }] },
|
|
{ items: [{ ...item, text: 8 }] },
|
|
{ items: [{ ...item, start: 3, end: 4 }] },
|
|
])
|
|
assert.throws(() => validateAnnotations(value, source, 0, source.length));
|
|
assert.throws(() => validateAnnotations({ items: Array(181).fill(item) }, source, 0, source.length));
|
|
assert.throws(() => validateAnnotations({ items: [item] }, source, 2, source.length));
|
|
assert.equal(processors.definition.strategy, "focus");
|
|
assert.equal(processors.grammar.strategy, "section");
|
|
assert.deepEqual(processors.punctuation.local!("你好。", 4), [
|
|
{ start: 6, end: 7, text: "Sentence boundary", tag: "punctuation" },
|
|
]);
|
|
console.log("PASS structured annotation validation, scope limits and processor strategies");
|
|
|
|
import { normalizeConfig, fieldsFor, underlineControls, labelControls, semanticConfig } from "../src/config.js";
|
|
const defaultOptions = normalizeConfig(processors.grammar, "{}");
|
|
assert.equal(defaultOptions.nounColor, "#6a8aac");
|
|
assert.equal(fieldsFor(processors.grammar).length, 6);
|
|
assert(!Object.hasOwn(defaultOptions, "underlinePattern"));
|
|
assert(!Object.hasOwn(defaultOptions, "fontSize"));
|
|
for (const options of [
|
|
'{"underlinePattern":"dashed"}',
|
|
'{"placement":"below"}',
|
|
'{"fontSize":24}',
|
|
'{"fontFamily":"monospace"}',
|
|
])
|
|
assert.throws(() => normalizeConfig(processors.grammar, options), /Unknown annotation setting/);
|
|
const grouped = {
|
|
...processors.punctuation,
|
|
config: [
|
|
underlineControls("primary", "Primary", { pattern: "dashed" }),
|
|
underlineControls("secondary", "Secondary"),
|
|
labelControls("label", "Labels"),
|
|
{
|
|
key: "meaning",
|
|
label: "Meaning",
|
|
kind: "group" as const,
|
|
fields: [{ key: "language", label: "Language", kind: "text" as const, default: "English" }],
|
|
},
|
|
],
|
|
};
|
|
const options = normalizeConfig(
|
|
grouped,
|
|
JSON.stringify({
|
|
primary: { underlinePattern: "squiggly", color: "#112233" },
|
|
secondary: { underlinePattern: "solid" },
|
|
label: { placement: "below", fontSize: 24 },
|
|
meaning: { language: "French" },
|
|
}),
|
|
);
|
|
assert.deepEqual(options.primary, { color: "#112233", underlinePattern: "squiggly" });
|
|
assert.deepEqual(options.secondary, { color: "#66866b", underlinePattern: "solid" });
|
|
assert.deepEqual(semanticConfig(grouped, options), { meaning: { language: "French" } });
|
|
assert.throws(() => normalizeConfig(grouped, '{"primary":{"underlinePattern":"wavy"}}'), /primary.Pattern/);
|
|
assert.throws(() => normalizeConfig(grouped, '{"primary":{"color":"red"}}'), /primary.Color/);
|
|
assert.throws(() => normalizeConfig(grouped, '{"primary":"dashed"}'), /Expected annotation settings/);
|
|
assert.throws(() => normalizeConfig(grouped, '{"primary":{"unknown":true}}'), /primary.unknown/);
|
|
assert.throws(() => normalizeConfig(grouped, '{"label":{"fontSize":0}}'), /label.Font size/);
|
|
assert.throws(() => normalizeConfig(grouped, '{"label":{"placement":"under"}}'), /label.Position/);
|
|
assert.throws(
|
|
() =>
|
|
normalizeConfig({ ...grouped, config: [underlineControls("same", "One"), underlineControls("same", "Two")] }, "{}"),
|
|
/Duplicate/,
|
|
);
|
|
const defaultLabel = normalizeConfig(processors.pinyin, '{"label":{"placement":"below"}}').label;
|
|
assert.equal(typeof defaultLabel === "object" && defaultLabel.placement, "below");
|
|
assert.equal(processors.pinyin.prompt, undefined);
|
|
assert.deepEqual(fieldsFor(processors.definition), [], "Fixed sidebar annotators expose no invented controls");
|
|
assert.deepEqual(normalizeConfig(processors.definitionTooltip, "{}"), { reveal: "hover" });
|
|
assert.deepEqual(normalizeConfig(processors.definitionTooltip, '{"reveal":"click"}'), { reveal: "click" });
|
|
assert.deepEqual(semanticConfig(processors.definitionTooltip, { reveal: "click" }), {});
|
|
assert.throws(() => normalizeConfig(processors.definitionTooltip, '{"reveal":"always"}'), /Show definition/);
|
|
console.log("PASS generated annotator config, validation, prompt options and dynamic typography/color");
|
|
|
|
import { TextIndex, processingPolicy, textIndex } from "../src/text-processing.js";
|
|
const policy = processingPolicy();
|
|
const article = Array.from({ length: 300 }, (_, n) => `第${n}句，阅读👩🏽💻和e\u0301。下一句提供上下文。\n`).join("");
|
|
const index = new TextIndex(article),
|
|
chunks = index.chunks(policy.chunkChars);
|
|
assert.equal(chunks.map((range) => article.slice(range.start, range.end)).join(""), article);
|
|
assert.equal(chunks[0].start, 0);
|
|
assert.equal(chunks.at(-1)!.end, article.length);
|
|
for (const range of chunks) {
|
|
assert.deepEqual(index.range(range), range);
|
|
assert(new TextIndex(article.slice(range.start, range.end)).units.length <= policy.chunkChars);
|
|
}
|
|
for (let start = 0; start < article.length; start += 57) {
|
|
const window = { start, end: Math.min(article.length, start + 500) };
|
|
const nearby = index.nearby(window, { start, end: start + 100 }, policy);
|
|
assert(nearby.length <= 3);
|
|
assert(
|
|
nearby.every((chunk) => chunks.includes(chunk)),
|
|
"Overlapping viewports reuse stable chunk identities",
|
|
);
|
|
assert(nearby.every((chunk) => chunk.end > window.start && chunk.start < window.end));
|
|
const context = index.context(window, policy);
|
|
assert(new TextIndex(article.slice(context.start, context.end)).units.length <= policy.contextChars);
|
|
}
|
|
const sentences = new TextIndex(
|
|
"First sentence. Second sentence. Third sentence. Fourth sentence. Fifth sentence. Sixth sentence.",
|
|
);
|
|
const fourth = sentences.sentences[3];
|
|
assert.deepEqual(sentences.focus({ start: fourth.start + 4, end: fourth.start + 5 }, "sentence", policy), fourth);
|
|
assert.equal(sentences.context(fourth, policy).start, sentences.sentences[1].start);
|
|
assert.equal(sentences.context(fourth, policy).end, sentences.text.length);
|
|
const huge = new TextIndex("无".repeat(10000));
|
|
for (const scope of ["word", "sentence", "section"]) {
|
|
const focused = huge.focus({ start: 5000, end: 9900 }, scope, policy);
|
|
assert(focused.end - focused.start <= 400, "No document-sized click, even without punctuation");
|
|
assert(focused.start <= 5000 && focused.end > 5000);
|
|
assert(huge.context(focused, policy).end - huge.context(focused, policy).start <= 1600);
|
|
}
|
|
assert.equal(
|
|
huge
|
|
.chunks(800)
|
|
.map((range) => huge.text.slice(range.start, range.end))
|
|
.join(""),
|
|
huge.text,
|
|
);
|
|
const emoji = new TextIndex("👩🏽💻e\u0301汉字");
|
|
assert.deepEqual(emoji.range({ start: 1, end: 2 }), { start: 0, end: 7 });
|
|
assert.deepEqual(emoji.tokens({ start: 7, end: 9 }), [{ start: 7, end: 9, text: "e\u0301" }]);
|
|
const longRange = processingPolicy({
|
|
context: "document",
|
|
contextChars: 80000,
|
|
focusChars: 2000,
|
|
chunkChars: 3000,
|
|
prefetchChunks: 2,
|
|
});
|
|
assert.deepEqual(index.context({ start: 1000, end: 1001 }, longRange), { start: 0, end: article.length });
|
|
assert(
|
|
huge.focus({ start: 5000, end: 5001 }, "sentence", longRange).end -
|
|
huge.focus({ start: 5000, end: 5001 }, "sentence", longRange).start >
|
|
400,
|
|
);
|
|
assert(index.nearby({ start: 3000, end: 3100 }, { start: 3000, end: 3100 }, longRange).length > 1);
|
|
for (const overrides of [
|
|
{ chunkChars: 0 },
|
|
{ focusChars: 80001 },
|
|
{ contextChars: NaN },
|
|
{ beforeSentences: -1 },
|
|
{ prefetchChunks: 21 },
|
|
])
|
|
assert.throws(() => processingPolicy(overrides));
|
|
assert.equal(textIndex(article), textIndex(article));
|
|
assert.equal(processors.passageTranslation.strategy, "section");
|
|
console.log(
|
|
"PASS reusable text planner: multilingual source offsets, stable bounded chunks, sentence context, pathological text, author overrides",
|
|
);
|
|
|
|
import { createVariant, nextSection, applySection, variantText } from "../src/variants.js";
|
|
const largeSource = "这是一个句子。\n".repeat(3000) + "👩🏽💻结束。";
|
|
let variant = createVariant(largeSource, "source-revision", "english");
|
|
assert(variant.sections.length > 10);
|
|
assert.equal(variant.sections.map((s) => largeSource.slice(s.start, s.end)).join(""), largeSource);
|
|
assert(
|
|
variant.sections.every(
|
|
(s) =>
|
|
Array.from(new Intl.Segmenter(undefined, { granularity: "grapheme" }).segment(largeSource.slice(s.start, s.end)))
|
|
.length <= 1400,
|
|
),
|
|
);
|
|
const first = nextSection(variant)!;
|
|
assert.throws(() => applySection(variant, first.start + 1, first.end, "bad"));
|
|
variant = applySection(variant, first.start, first.end, "A shorter section.");
|
|
const second = nextSection(variant)!;
|
|
variant = applySection(
|
|
variant,
|
|
second.start,
|
|
second.end,
|
|
"A much longer translated second section, with different output offsets.",
|
|
);
|
|
assert.equal(
|
|
variantText(variant),
|
|
"A shorter section.\n\nA much longer translated second section, with different output offsets.",
|
|
);
|
|
assert.equal(variant.sections[1].start, second.start, "Canonical section identities survive length changes");
|
|
console.log(
|
|
"PASS lazy variants: bounded Unicode sections, exact canonical coverage, sequential append and changing output lengths",
|
|
);
|
|
|
|
import { computationFields } from "../src/config.js";
|
|
import { cacheConfiguration, findCachedJob, sameCacheConfiguration } from "../src/cache.js";
|
|
const cached = (processor: typeof processors.pinyin, options: Record<string, unknown>) =>
|
|
cacheConfiguration("pinyin", "word", "section", options, computationFields(processor));
|
|
assert.equal(
|
|
cached(processors.pinyin, { minimumLevel: "2", fontSize: 13 }),
|
|
cached(processors.pinyin, { minimumLevel: "8", fontSize: 32 }),
|
|
"HSK/font display changes do not bust computation",
|
|
);
|
|
assert.deepEqual(computationFields(processors.names), [], "Per-category colors never reach inference");
|
|
assert.equal(
|
|
cached(grouped, options),
|
|
cached(grouped, normalizeConfig(grouped, '{"meaning":{"language":"French"}}')),
|
|
"Independent nested appearance groups preserve semantic caches",
|
|
);
|
|
assert.notEqual(
|
|
cached(grouped, options),
|
|
cached(grouped, normalizeConfig(grouped, "{}")),
|
|
"Nested semantic fields isolate caches",
|
|
);
|
|
const custom = { ...processors.pinyin, cache: { inputs: ["targetLanguage"] } };
|
|
assert.notEqual(
|
|
cached(custom, { targetLanguage: "English" }),
|
|
cached(custom, { targetLanguage: "French" }),
|
|
"Semantic inputs isolate caches",
|
|
);
|
|
assert.equal(
|
|
cacheConfiguration("pinyin", "word", "focus", {}, []),
|
|
cached(processors.pinyin, {}),
|
|
"Generation timing is independent of semantic results",
|
|
);
|
|
console.log("PASS semantic cache configuration, display-only HSK/color/font edits and explicit author inputs");
|
|
|
|
let activeReads = 0,
|
|
maximumReads = 0;
|
|
const history = Array.from({ length: 97 }, (_, index) => index);
|
|
const read = async (index: number) => {
|
|
maximumReads = Math.max(maximumReads, ++activeReads);
|
|
await Promise.resolve();
|
|
activeReads--;
|
|
return { key: index === 34 || index === 79 ? "wanted" : "other", status: index === 34 ? "failed" : "ready" };
|
|
};
|
|
assert.deepEqual(await findCachedJob(history, read, "wanted", true), { reference: undefined, retries: 2 });
|
|
assert(maximumReads > 1 && maximumReads <= 32, "Historical cache reads run concurrently with a bound");
|
|
assert.deepEqual(await findCachedJob(history, read, "wanted", false), { reference: 79, retries: 2 });
|
|
assert.deepEqual(await findCachedJob(history, read, "missing", true), { reference: undefined, retries: 0 });
|
|
console.log("PASS bounded cache admission, stable reuse order and retry accounting");
|
|
|
|
import { processorIdentity } from "../src/processor-identity.js";
|
|
const identity = (processor: typeof processors.pinyin, source = "package@1", task = "task@1") =>
|
|
processorIdentity("pinyin", processor, {}, source, task);
|
|
assert.deepEqual(
|
|
identity(processors.pinyin),
|
|
identity(processors.pinyin, "package@2"),
|
|
"Unrelated runtime changes retain declarative model results",
|
|
);
|
|
assert.deepEqual(
|
|
identity(processors.pinyin),
|
|
identity({ ...processors.pinyin, renders: [{ style: "tooltip", reveal: "hover" }] }),
|
|
"Render hooks do not invalidate inference",
|
|
);
|
|
assert.notDeepEqual(identity(processors.pinyin), identity({ ...processors.pinyin, instructions: "Different task" }));
|
|
assert.notDeepEqual(
|
|
identity(processors.pinyin),
|
|
identity({ ...processors.pinyin, processing: { contextChars: 800 } }),
|
|
"Context policy changes invalidate previously rendered results",
|
|
);
|
|
assert.deepEqual(
|
|
identity({ ...processors.pinyin, processing: { contextChars: 800, chunkChars: 500, prefetchChunks: 1 } }),
|
|
identity({ ...processors.pinyin, processing: { contextChars: 800, chunkChars: 900, prefetchChunks: 3 } }),
|
|
"Chunking and prefetch preferences do not invalidate semantic results",
|
|
);
|
|
assert.deepEqual(
|
|
identity(processors.pinyin),
|
|
identity({ ...processors.pinyin, profile: "cerebras-light" }),
|
|
"Routing preferences do not discard acceptable semantic results",
|
|
);
|
|
const configuredProcessor = {
|
|
...processors.pinyin,
|
|
config: [{ key: "targetLanguage", label: "Language", kind: "text" as const, default: "English" }],
|
|
};
|
|
assert.deepEqual(
|
|
identity(configuredProcessor),
|
|
identity({
|
|
...configuredProcessor,
|
|
config: [{ ...configuredProcessor.config[0], label: "Translate into", default: "French" }],
|
|
}),
|
|
"Config labels and defaults do not alter an already chosen semantic value",
|
|
);
|
|
assert.deepEqual(
|
|
identity(processors.pinyin),
|
|
identity(processors.pinyin, "package@1", "task@2"),
|
|
"Actual tokens/context key the result; unrelated planner edits do not erase semantic identity",
|
|
);
|
|
const localProcessor = { ...processors.pinyin, local: () => [] };
|
|
assert.notDeepEqual(
|
|
identity(localProcessor),
|
|
identity(localProcessor, "package@2"),
|
|
"Imported local helpers remain part of identity",
|
|
);
|
|
const versioned = { ...localProcessor, cache: { version: "semantic@1" } };
|
|
assert.deepEqual(identity(versioned), identity(versioned, "package@2"));
|
|
assert.notDeepEqual(identity(versioned), identity({ ...versioned, cache: { version: "semantic@2" } }));
|
|
assert.notDeepEqual(
|
|
identity(versioned),
|
|
identity({ ...versioned, local: () => [{ start: 0, end: 1, text: "changed", tag: "" }] }),
|
|
);
|
|
assert(
|
|
sameCacheConfiguration(
|
|
cached(processors.pinyin, {}),
|
|
JSON.stringify({ processor: "pinyin", scope: "word", generation: "focus", options: {} }),
|
|
),
|
|
);
|
|
assert(
|
|
!sameCacheConfiguration(cached(custom, { targetLanguage: "French" }), cached(custom, { targetLanguage: "English" })),
|
|
);
|
|
assert(!sameCacheConfiguration("malformed", "also malformed"));
|
|
console.log(
|
|
"PASS semantic processor revisions, routing-independent results, explicit code versions and legacy scheduling normalization",
|
|
);
|
|
|
|
const mismatched = [
|
|
{ ...item, source: "欢迎" },
|
|
{ ...item, start: 2, end: 9, source: "wrong" },
|
|
];
|
|
try {
|
|
validateAnnotations({ items: mismatched }, source, 0, source.length);
|
|
assert.fail("Expected source mismatches");
|
|
} catch (error) {
|
|
assert(error instanceof AnnotationValidationError);
|
|
assert.equal(error.errors.length, 2);
|
|
assert(error.errors[0].includes("items[0].source"));
|
|
assert(error.errors[1].includes("items[1].source"));
|
|
}
|
|
assert.throws(() => validateAnnotations({ items: [renderedItem] }, source, 0, source.length), /source/);
|
|
assert.deepEqual(validateAnnotations({ items: [renderedItem] }, source, 0, source.length, false), [renderedItem]);
|
|
assert(annotationSchemaFor(processors.pinyin).properties.items.items.required.includes("source"));
|
|
assert(
|
|
!annotationSchemaFor({ ...processors.pinyin, verifySource: false }).properties.items.items.required.includes(
|
|
"source",
|
|
),
|
|
);
|
|
const repeated = "café café 👩🏽💻 e\u0301";
|
|
assert.deepEqual(
|
|
validateAnnotations({ items: [{ start: 5, end: 9, source: "café", text: "coffee", tag: "noun" }] }, repeated, 5, 9),
|
|
[{ start: 5, end: 9, text: "coffee", tag: "noun" }],
|
|
);
|
|
assert.throws(
|
|
() =>
|
|
validateAnnotations(
|
|
{ items: [{ start: 10, end: 17, source: "👩", text: "developer", tag: "noun" }] },
|
|
repeated,
|
|
0,
|
|
repeated.length,
|
|
),
|
|
/source/,
|
|
);
|
|
console.log("PASS exact source echo, all indexed mismatches, repeated words, UTF-16 and explicit code-owned opt-out");
|
|
|
|
try {
|
|
validateAnnotations(
|
|
{
|
|
items: Array.from({ length: 180 }, () => ({
|
|
start: 0,
|
|
end: 120,
|
|
source: "\u0002".repeat(120),
|
|
text: "meaning",
|
|
tag: "",
|
|
})),
|
|
},
|
|
"\u0001".repeat(120),
|
|
0,
|
|
120,
|
|
);
|
|
assert.fail("Expected all mismatches");
|
|
} catch (error) {
|
|
assert(error instanceof AnnotationValidationError);
|
|
assert.equal(error.errors.length, 180);
|
|
assert(
|
|
new TextEncoder().encode(JSON.stringify(error.errors)).length < 49152,
|
|
"Every located error fits in one bounded feedback turn",
|
|
);
|
|
}
|