package-language-reader-runtime/tests/processors.test.ts

import assert from "node:assert/strict";
import { processors, validateAnnotations, AnnotationValidationError, annotationSchemaFor } from "../src/processors.js";
const source = "北京👩🏽‍💻欢迎你";
const item = { start: 0, end: 2, source: "北京", text: "Běijīng", tag: "place" };
const renderedItem = { start: 0, end: 2, text: "Běijīng", tag: "place" };
assert.deepEqual(validateAnnotations({ items: [item] }, source, 0, source.length), [renderedItem]);
for (const value of [
  null,
  { items: [null] },
  { items: [{ ...item, end: 999 }] },
  { items: [{ ...item, start: 1, end: 0 }] },
  { items: [{ ...item, extra: true }] },
  { items: [{ ...item, text: 8 }] },
  { items: [{ ...item, start: 3, end: 4 }] },
])
  assert.throws(() => validateAnnotations(value, source, 0, source.length));
assert.throws(() => validateAnnotations({ items: Array(181).fill(item) }, source, 0, source.length));
assert.throws(() => validateAnnotations({ items: [item] }, source, 2, source.length));
assert.equal(processors.definition.strategy, "focus");
assert.equal(processors.grammar.strategy, "section");
assert.deepEqual(processors.punctuation.local!("你好。", 4), [
  { start: 6, end: 7, text: "Sentence boundary", tag: "punctuation" },
]);
console.log("PASS structured annotation validation, scope limits and processor strategies");

import { normalizeConfig, fieldsFor } from "../src/config.js";
const defaultOptions = normalizeConfig(processors.grammar, "{}");
assert.equal(defaultOptions.nounColor, "#6a8aac");
assert.equal(fieldsFor(processors.grammar).length, 6);
const customized = normalizeConfig(
  processors.grammar,
  JSON.stringify({ nounColor: "#112233", fontFamily: "monospace", fontSize: 24 }),
);
assert.equal(customized.fontSize, 24);
assert.equal(defaultOptions.underlinePattern, "solid");
assert.equal(defaultOptions.placement, "above");
assert.equal(normalizeConfig(processors.pinyin, '{"placement":"below"}').placement, "below");
for (const placement of ["under", "", null, 1])
  assert.throws(() => normalizeConfig(processors.pinyin, JSON.stringify({ placement })), /Label position/);
for (const underlinePattern of ["solid", "dashed", "squiggly"]) {
  assert.equal(
    normalizeConfig(processors.grammar, JSON.stringify({ underlinePattern })).underlinePattern,
    underlinePattern,
  );
}
for (const underlinePattern of ["wavy", "", null, 1]) {
  assert.throws(() => normalizeConfig(processors.grammar, JSON.stringify({ underlinePattern })), /Underline pattern/);
}
for (const options of [
  "[]",
  "null",
  '{"unknown":true}',
  '{"fontSize":0}',
  '{"fontSize":null}',
  '{"fontSize":80}',
  '{"fontSize":"24"}',
  '{"nounColor":"red"}',
  '{"minimumLevel":"not-a-level"}',
  '{"fontFamily":"bad\\nfont"}',
]) {
  assert.throws(() =>
    normalizeConfig(options.includes("minimumLevel") ? processors.pinyin : processors.grammar, options),
  );
}
assert.equal(processors.pinyin.prompt, undefined);
console.log("PASS generated annotator config, validation, prompt options and dynamic typography/color");

import { TextIndex, processingPolicy, textIndex } from "../src/text-processing.js";
const policy = processingPolicy();
const article = Array.from({ length: 300 }, (_, n) => `第${n}句，阅读👩🏽‍💻和e\u0301。下一句提供上下文。\n`).join("");
const index = new TextIndex(article),
  chunks = index.chunks(policy.chunkChars);
assert.equal(chunks.map((range) => article.slice(range.start, range.end)).join(""), article);
assert.equal(chunks[0].start, 0);
assert.equal(chunks.at(-1)!.end, article.length);
for (const range of chunks) {
  assert.deepEqual(index.range(range), range);
  assert(new TextIndex(article.slice(range.start, range.end)).units.length <= policy.chunkChars);
}
for (let start = 0; start < article.length; start += 57) {
  const window = { start, end: Math.min(article.length, start + 500) };
  const nearby = index.nearby(window, { start, end: start + 100 }, policy);
  assert(nearby.length <= 3);
  assert(
    nearby.every((chunk) => chunks.includes(chunk)),
    "Overlapping viewports reuse stable chunk identities",
  );
  assert(nearby.every((chunk) => chunk.end > window.start && chunk.start < window.end));
  const context = index.context(window, policy);
  assert(new TextIndex(article.slice(context.start, context.end)).units.length <= policy.contextChars);
}
const sentences = new TextIndex(
  "First sentence. Second sentence. Third sentence. Fourth sentence. Fifth sentence. Sixth sentence.",
);
const fourth = sentences.sentences[3];
assert.deepEqual(sentences.focus({ start: fourth.start + 4, end: fourth.start + 5 }, "sentence", policy), fourth);
assert.equal(sentences.context(fourth, policy).start, sentences.sentences[1].start);
assert.equal(sentences.context(fourth, policy).end, sentences.text.length);
const huge = new TextIndex("无".repeat(10000));
for (const scope of ["word", "sentence", "section"]) {
  const focused = huge.focus({ start: 5000, end: 9900 }, scope, policy);
  assert(focused.end - focused.start <= 400, "No document-sized click, even without punctuation");
  assert(focused.start <= 5000 && focused.end > 5000);
  assert(huge.context(focused, policy).end - huge.context(focused, policy).start <= 1600);
}
assert.equal(
  huge
    .chunks(800)
    .map((range) => huge.text.slice(range.start, range.end))
    .join(""),
  huge.text,
);
const emoji = new TextIndex("👩🏽‍💻e\u0301汉字");
assert.deepEqual(emoji.range({ start: 1, end: 2 }), { start: 0, end: 7 });
assert.deepEqual(emoji.tokens({ start: 7, end: 9 }), [{ start: 7, end: 9, text: "e\u0301" }]);
const longRange = processingPolicy({
  context: "document",
  contextChars: 80000,
  focusChars: 2000,
  chunkChars: 3000,
  prefetchChunks: 2,
});
assert.deepEqual(index.context({ start: 1000, end: 1001 }, longRange), { start: 0, end: article.length });
assert(
  huge.focus({ start: 5000, end: 5001 }, "sentence", longRange).end -
    huge.focus({ start: 5000, end: 5001 }, "sentence", longRange).start >
    400,
);
assert(index.nearby({ start: 3000, end: 3100 }, { start: 3000, end: 3100 }, longRange).length > 1);
for (const overrides of [
  { chunkChars: 0 },
  { focusChars: 80001 },
  { contextChars: NaN },
  { beforeSentences: -1 },
  { prefetchChunks: 21 },
])
  assert.throws(() => processingPolicy(overrides));
assert.equal(textIndex(article), textIndex(article));
assert.equal(processors.passageTranslation.strategy, "section");
console.log(
  "PASS reusable text planner: multilingual source offsets, stable bounded chunks, sentence context, pathological text, author overrides",
);

import { createVariant, nextSection, applySection, variantText } from "../src/variants.js";
const largeSource = "这是一个句子。\n".repeat(3000) + "👩🏽‍💻结束。";
let variant = createVariant(largeSource, "source-revision", "english");
assert(variant.sections.length > 10);
assert.equal(variant.sections.map((s) => largeSource.slice(s.start, s.end)).join(""), largeSource);
assert(
  variant.sections.every(
    (s) =>
      Array.from(new Intl.Segmenter(undefined, { granularity: "grapheme" }).segment(largeSource.slice(s.start, s.end)))
        .length <= 1400,
  ),
);
const first = nextSection(variant)!;
assert.throws(() => applySection(variant, first.start + 1, first.end, "bad"));
variant = applySection(variant, first.start, first.end, "A shorter section.");
const second = nextSection(variant)!;
variant = applySection(
  variant,
  second.start,
  second.end,
  "A much longer translated second section, with different output offsets.",
);
assert.equal(
  variantText(variant),
  "A shorter section.\n\nA much longer translated second section, with different output offsets.",
);
assert.equal(variant.sections[1].start, second.start, "Canonical section identities survive length changes");
console.log(
  "PASS lazy variants: bounded Unicode sections, exact canonical coverage, sequential append and changing output lengths",
);

import { computationFields } from "../src/config.js";
import { cacheConfiguration, findCachedJob, sameCacheConfiguration } from "../src/cache.js";
const cached = (processor: typeof processors.pinyin, options: Record<string, unknown>) =>
  cacheConfiguration("pinyin", "word", "section", options, computationFields(processor));
assert.equal(
  cached(processors.pinyin, { minimumLevel: "2", fontSize: 13 }),
  cached(processors.pinyin, { minimumLevel: "8", fontSize: 32 }),
  "HSK/font display changes do not bust computation",
);
assert.deepEqual(computationFields(processors.names), [], "Per-category colors never reach inference");
assert.equal(
  cached(processors.grammar, normalizeConfig(processors.grammar, '{"underlinePattern":"solid"}')),
  cached(processors.grammar, normalizeConfig(processors.grammar, '{"underlinePattern":"squiggly"}')),
  "Changing underline geometry reuses the same computed annotation results",
);
assert.equal(
  cached(processors.pinyin, normalizeConfig(processors.pinyin, '{"underlinePattern":"dashed"}')),
  cached(processors.pinyin, normalizeConfig(processors.pinyin, "{}")),
  "Common underline appearance never reaches semantic prompt configuration",
);
const custom = { ...processors.pinyin, cache: { inputs: ["targetLanguage"] } };
assert.notEqual(
  cached(custom, { targetLanguage: "English" }),
  cached(custom, { targetLanguage: "French" }),
  "Semantic inputs isolate caches",
);
assert.equal(
  cacheConfiguration("pinyin", "word", "focus", {}, []),
  cached(processors.pinyin, {}),
  "Generation timing is independent of semantic results",
);
console.log("PASS semantic cache configuration, display-only HSK/color/font edits and explicit author inputs");

let activeReads = 0,
  maximumReads = 0;
const history = Array.from({ length: 97 }, (_, index) => index);
const read = async (index: number) => {
  maximumReads = Math.max(maximumReads, ++activeReads);
  await Promise.resolve();
  activeReads--;
  return { key: index === 34 || index === 79 ? "wanted" : "other", status: index === 34 ? "failed" : "ready" };
};
assert.deepEqual(await findCachedJob(history, read, "wanted", true), { reference: undefined, retries: 2 });
assert(maximumReads > 1 && maximumReads <= 32, "Historical cache reads run concurrently with a bound");
assert.deepEqual(await findCachedJob(history, read, "wanted", false), { reference: 79, retries: 2 });
assert.deepEqual(await findCachedJob(history, read, "missing", true), { reference: undefined, retries: 0 });
console.log("PASS bounded cache admission, stable reuse order and retry accounting");

import { processorIdentity } from "../src/processor-identity.js";
const identity = (processor: typeof processors.pinyin, source = "package@1", task = "task@1") =>
  processorIdentity("pinyin", processor, {}, source, task);
assert.deepEqual(
  identity(processors.pinyin),
  identity(processors.pinyin, "package@2"),
  "Unrelated runtime changes retain declarative model results",
);
assert.deepEqual(
  identity(processors.pinyin),
  identity({ ...processors.pinyin, renders: [{ style: "tooltip", reveal: "hover" }] }),
  "Render hooks do not invalidate inference",
);
assert.notDeepEqual(identity(processors.pinyin), identity({ ...processors.pinyin, instructions: "Different task" }));
assert.notDeepEqual(
  identity(processors.pinyin),
  identity({ ...processors.pinyin, processing: { contextChars: 800 } }),
  "Context policy changes invalidate previously rendered results",
);
assert.deepEqual(
  identity({ ...processors.pinyin, processing: { contextChars: 800, chunkChars: 500, prefetchChunks: 1 } }),
  identity({ ...processors.pinyin, processing: { contextChars: 800, chunkChars: 900, prefetchChunks: 3 } }),
  "Chunking and prefetch preferences do not invalidate semantic results",
);
assert.deepEqual(
  identity(processors.pinyin),
  identity({ ...processors.pinyin, profile: "cerebras-light" }),
  "Routing preferences do not discard acceptable semantic results",
);
const configuredProcessor = {
  ...processors.pinyin,
  config: [{ key: "targetLanguage", label: "Language", kind: "text" as const, default: "English" }],
};
assert.deepEqual(
  identity(configuredProcessor),
  identity({
    ...configuredProcessor,
    config: [{ ...configuredProcessor.config[0], label: "Translate into", default: "French" }],
  }),
  "Config labels and defaults do not alter an already chosen semantic value",
);
assert.deepEqual(
  identity(processors.pinyin),
  identity(processors.pinyin, "package@1", "task@2"),
  "Actual tokens/context key the result; unrelated planner edits do not erase semantic identity",
);
const localProcessor = { ...processors.pinyin, local: () => [] };
assert.notDeepEqual(
  identity(localProcessor),
  identity(localProcessor, "package@2"),
  "Imported local helpers remain part of identity",
);
const versioned = { ...localProcessor, cache: { version: "semantic@1" } };
assert.deepEqual(identity(versioned), identity(versioned, "package@2"));
assert.notDeepEqual(identity(versioned), identity({ ...versioned, cache: { version: "semantic@2" } }));
assert.notDeepEqual(
  identity(versioned),
  identity({ ...versioned, local: () => [{ start: 0, end: 1, text: "changed", tag: "" }] }),
);
assert(
  sameCacheConfiguration(
    cached(processors.pinyin, {}),
    JSON.stringify({ processor: "pinyin", scope: "word", generation: "focus", options: {} }),
  ),
);
assert(
  !sameCacheConfiguration(cached(custom, { targetLanguage: "French" }), cached(custom, { targetLanguage: "English" })),
);
assert(!sameCacheConfiguration("malformed", "also malformed"));
console.log(
  "PASS semantic processor revisions, routing-independent results, explicit code versions and legacy scheduling normalization",
);

const mismatched = [
  { ...item, source: "欢迎" },
  { ...item, start: 2, end: 9, source: "wrong" },
];
try {
  validateAnnotations({ items: mismatched }, source, 0, source.length);
  assert.fail("Expected source mismatches");
} catch (error) {
  assert(error instanceof AnnotationValidationError);
  assert.equal(error.errors.length, 2);
  assert(error.errors[0].includes("items[0].source"));
  assert(error.errors[1].includes("items[1].source"));
}
assert.throws(() => validateAnnotations({ items: [renderedItem] }, source, 0, source.length), /source/);
assert.deepEqual(validateAnnotations({ items: [renderedItem] }, source, 0, source.length, false), [renderedItem]);
assert(annotationSchemaFor(processors.pinyin).properties.items.items.required.includes("source"));
assert(
  !annotationSchemaFor({ ...processors.pinyin, verifySource: false }).properties.items.items.required.includes(
    "source",
  ),
);
const repeated = "café café 👩🏽‍💻 e\u0301";
assert.deepEqual(
  validateAnnotations({ items: [{ start: 5, end: 9, source: "café", text: "coffee", tag: "noun" }] }, repeated, 5, 9),
  [{ start: 5, end: 9, text: "coffee", tag: "noun" }],
);
assert.throws(
  () =>
    validateAnnotations(
      { items: [{ start: 10, end: 17, source: "👩", text: "developer", tag: "noun" }] },
      repeated,
      0,
      repeated.length,
    ),
  /source/,
);
console.log("PASS exact source echo, all indexed mismatches, repeated words, UTF-16 and explicit code-owned opt-out");

try {
  validateAnnotations(
    {
      items: Array.from({ length: 180 }, () => ({
        start: 0,
        end: 120,
        source: "\u0002".repeat(120),
        text: "meaning",
        tag: "",
      })),
    },
    "\u0001".repeat(120),
    0,
    120,
  );
  assert.fail("Expected all mismatches");
} catch (error) {
  assert(error instanceof AnnotationValidationError);
  assert.equal(error.errors.length, 180);
  assert(
    new TextEncoder().encode(JSON.stringify(error.errors)).length < 49152,
    "Every located error fits in one bounded feedback turn",
  );
}