commit b4d56d0eb98a5d18eb68b9c460830b3503760e17 Author: Quixos Template Publisher Date: Sat Jan 1 00:00:00 2000 +0000 Export template source from e33714a5ca38429d1bdb77a961a14461697944f5:examples/local-first/packages/gutenberg-runtime diff --git a/.quixos-template-source.json b/.quixos-template-source.json new file mode 100644 index 0000000..b2c8eff --- /dev/null +++ b/.quixos-template-source.json @@ -0,0 +1,4 @@ +{ + "monorepoCommit": "e33714a5ca38429d1bdb77a961a14461697944f5", + "sourcePath": "examples/local-first/packages/gutenberg-runtime" +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..eadbaf4 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +# Project Gutenberg source for Reader + +Updated September 30, 2026. The catalog is an ordinary Camino entity with +`search` and `readBook` remote operations. It has an independent Studio +presentation and an embeddable `GutenbergPicker`. Reader's document-author +adapter imports the chosen excerpt through the existing durable publication +connection; completion navigates the bookshelf to the new document. Manual +text authoring keeps its own independent state. + +## Catalog and downloads + +`catalog.csv.gz` contains the reviewed September 30, 2026 snapshot of +Gutenberg's machine-readable CSV. Its SHA-256 is +`62a53fdce11252795b8f247d4cdb51d100a2cf4856a716ff59d53959ad1c19c3`. +The build parses text editions and bundles the index into the service. Fuzzy +search over titles/authors reads that index without contacting Gutenberg. +Replace the snapshot deliberately and repin workspace resources when refreshing +it. The daily feed is not a build-time dependency: older source revisions must +remain buildable even after it changes. The index builder checks coverage and +representative titles before accepting a snapshot. + +Book text is requested when a user opens a result, from the +[listed generated-files mirror](https://www.gutenberg.org/dirs/MIRRORS.ALL) +`gutenberg.pglaf.org`, following the +[robot access policy](https://www.gutenberg.org/policy/robot_access.html). +The downloader reads only far enough to collect about 1,000 words, ending at +the next complete paragraph, then cancels the remaining response body. Gutenberg +header/footer markers are removed. Short books end at EOF. Exceptionally long +paragraphs are bounded at 56,000 characters, preferring a word boundary and +preserving Unicode surrogate pairs. + +`readBook({ebook})` returns `{book, url, text, words}` in one response. The picker +shows a preview and an **Import excerpt** button; the standalone catalog offers +**Copy excerpt**. There is no paragraph pagination or selection step. + +Downloads are serialized, start at least two seconds apart, and coalesce requests +for the same book. The process caches eight excerpts, bounded by 32 MiB. The UI +also keeps eight excerpts while mounted, so going back to a book needs no new +remote call. The first request still needs the mirror to respond. Downloads have +a 20-second timeout and 12 MiB maximum; missing editions and failures can be +retried. Search never downloads books. + +## Verification + +```sh +node --test examples/local-first/packages/gutenberg-runtime/test/catalog.test.mjs +nix run .#camino-presentation-qualification +``` + +Unit tests cover paragraph/Unicode boundaries, truncated streams, stopping and +cancelling a download, coalescing, caching, serialization and retries. Browser +checks search the actual pinned index and stub only the external book response. diff --git a/build-index.mjs b/build-index.mjs new file mode 100644 index 0000000..00dc11e --- /dev/null +++ b/build-index.mjs @@ -0,0 +1,11 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { parseIndex } from "./src/catalog.ts"; +const books = parseIndex(await readFile(process.argv[2], "utf8")); +if ( + books.length < 70000 || + !books.some((b) => b.ebook === 1342 && b.title.includes("Pride")) || + !books.some((b) => b.ebook === 11 && b.title.includes("Alice")) +) + throw Error("Gutenberg CSV format/coverage changed; review parser before updating pin"); +await writeFile(process.argv[3], "export default " + JSON.stringify(books) + ";\n"); +console.log("Indexed", books.length, "Gutenberg titles"); diff --git a/catalog.csv.gz b/catalog.csv.gz new file mode 100644 index 0000000..624ae6f Binary files /dev/null and b/catalog.csv.gz differ diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..54403f0 --- /dev/null +++ b/flake.lock @@ -0,0 +1,108 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "helpers": { + "flake": false, + "locked": { + "lastModified": 1790852599, + "narHash": "sha256-MdNtZpcsoguLPSCpAy9Naz6fkVPrftbZQbz1Aj7Zm9A=", + "ref": "refs/tags/quixos-reachability/91679e8cc82133093cb0b4f9e519cbb39a5bba4f", + "rev": "91679e8cc82133093cb0b4f9e519cbb39a5bba4f", + "revCount": 54, + "type": "git", + "url": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git" + }, + "original": { + "ref": "refs/tags/quixos-reachability/91679e8cc82133093cb0b4f9e519cbb39a5bba4f", + "rev": "91679e8cc82133093cb0b4f9e519cbb39a5bba4f", + "type": "git", + "url": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1783776592, + "narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "protocol": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1791261504, + "narHash": "sha256-QSk9DWR7n5KL6cMHzmSKKXNIyaFA0s8Nu+wXjhhlENI=", + "ref": "refs/tags/quixos-reachability/5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87", + "rev": "5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87", + "revCount": 184, + "type": "git", + "url": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-protocol.git" + }, + "original": { + "ref": "refs/tags/quixos-reachability/5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87", + "rev": "5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87", + "type": "git", + "url": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-protocol.git" + } + }, + "root": { + "inputs": { + "flake-utils": [ + "protocol", + "flake-utils" + ], + "helpers": "helpers", + "nixpkgs": [ + "protocol", + "nixpkgs" + ], + "protocol": "protocol" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..de1ed12 --- /dev/null +++ b/flake.nix @@ -0,0 +1,65 @@ +{ + inputs.protocol.url = "git+https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-protocol.git?ref=refs/tags/quixos-reachability/5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87&rev=5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87"; + inputs.nixpkgs.follows = "protocol/nixpkgs"; + inputs.flake-utils.follows = "protocol/flake-utils"; + inputs.helpers = { + url = "git+https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git?ref=refs/tags/quixos-reachability/91679e8cc82133093cb0b4f9e519cbb39a5bba4f&rev=91679e8cc82133093cb0b4f9e519cbb39a5bba4f"; + flake = false; + }; + outputs = + { + nixpkgs, + flake-utils, + helpers, + ... + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { inherit system; }; + camino = import (helpers + "/quixos-package-helpers.nix"); + in + { + quixosPackages.checkedArtifacts = + context: + let + pkgs = camino.packageSet { inherit nixpkgs system context; }; + # Gutenberg regenerates its feed daily. Keep the reviewed bytes in + # the source revision so a clean machine can still build old pins. + catalog = ./catalog.csv.gz; + source = + pkgs.runCommand "gutenberg-source" + { + nativeBuildInputs = [ + pkgs.nodejs_24 + pkgs.gzip + ]; + } + '' + cp -r ${./.} "$out" + chmod -R u+w "$out" + gzip -dc ${catalog} > catalog.csv + node "$out/build-index.mjs" catalog.csv "$out/src/index-data.ts" + ''; + portable = camino.mkCaminoTypeScriptPackage { + inherit pkgs context; + src = source; + entry = "src/portable.ts"; + }; + remote = camino.mkCaminoTypeScriptService { + inherit pkgs context; + src = source; + entry = "src/remote.ts"; + }; + in + camino.assembleCaminoPackage { + inherit pkgs context; + artifacts = [ + portable + remote + ]; + }; + devShells.default = camino.mkCaminoYarnShell { inherit pkgs; }; + } + ); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..2926178 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "name": "@quixos-example/gutenberg-runtime", + "private": true, + "type": "module" +} diff --git a/package.qx b/package.qx new file mode 100644 index 0000000..6cd28b1 --- /dev/null +++ b/package.qx @@ -0,0 +1,10 @@ +symbol JavaScript id "target:org.quixos.execution.javascript@1"; +external atom GutenbergCatalogObject id "atom:gutenberg:catalog"; +import interface GutenbergCatalog; +type GutenbergBook = record {ebook:uint32;title:string;author:string;language:string;}; +type GutenbergExcerpt = record {book:GutenbergBook;url:string;text:string;words:uint32;}; +package GutenbergRuntime id "package:gutenberg:runtime" revision "package:gutenberg:runtime@1" { + function create id "gutenberg:create" : unit -> atom-ref execution transaction targets [JavaScript,server] requires {creation catalog id "catalog" : GutenbergCatalogObject input record {name:string;};}; + operation search id "gutenberg:search" : string -> list mode call execution remote receiver any; + operation readBook id "gutenberg:read" : record {ebook:uint32;} -> GutenbergExcerpt mode call execution remote receiver any; +} diff --git a/quixos.lock b/quixos.lock new file mode 100644 index 0000000..f62abae --- /dev/null +++ b/quixos.lock @@ -0,0 +1,4 @@ +quixos-lock version 1 { + quixos source { repository "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos.git"; commit "e33714a5ca38429d1bdb77a961a14461697944f5"; } + interface GutenbergCatalog source { repository "https://repos.quixos.org/quixos-local-first-examples/interface-gutenberg-catalog.git"; commit "af88f1abc529f82f4464e4c0ed99cd2fff6b82ea"; } +} diff --git a/src/catalog.ts b/src/catalog.ts new file mode 100644 index 0000000..116a411 --- /dev/null +++ b/src/catalog.ts @@ -0,0 +1,131 @@ +export interface Book { + ebook: number; + title: string; + author: string; + language: string; +} +/** RFC 4180 records, including quoted commas, escaped quotes, and multiline titles. */ +export function parseIndex(text: string): Book[] { + const rows: string[][] = []; + let row: string[] = [], + field = "", + quoted = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]!; + if (ch === '"') { + if (quoted && text[i + 1] === '"') { + field += '"'; + i++; + } else if (quoted || field === "") quoted = !quoted; + else throw Error("Unexpected CSV quote"); + } else if (!quoted && (ch === "," || ch === "\n" || ch === "\r")) { + row.push(field); + field = ""; + if (ch !== ",") { + rows.push(row); + row = []; + if (ch === "\r" && text[i + 1] === "\n") i++; + } + } else field += ch; + } + if (quoted) throw Error("Unclosed CSV quote"); + if (field || row.length) { + row.push(field); + rows.push(row); + } + const header = rows.shift()!; + const columns = ["Text#", "Type", "Title", "Language", "Authors"].map((name) => header.indexOf(name)); + if (columns.some((i) => i < 0)) throw Error("Unrecognized Gutenberg CSV header"); + const books: Book[] = [], + seen = new Set(); + for (const row of rows) { + if (row.length === 1 && row[0] === "") continue; + if (row.length !== header.length) throw Error("Invalid Gutenberg CSV row"); + if (row[columns[1]!] !== "Text") continue; + const ebook = Number(row[columns[0]!]!); + if (!Number.isSafeInteger(ebook) || ebook < 1 || seen.has(ebook)) throw Error("Invalid or repeated book ID"); + seen.add(ebook); + books.push({ + ebook, + title: row[columns[2]!]!.replace(/\s+/g, " ").trim(), + language: row[columns[3]!]!, + author: row[columns[4]!]!, + }); + } + return books; +} +const normalized = (text: string) => + text + .normalize("NFKD") + .replace(/\p{M}/gu, "") + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, " ") + .trim(); +const grams = (text: string) => + new Set(Array.from({ length: Math.max(0, text.length - 2) }, (_, i) => text.slice(i, i + 3))); +export function searchIndex(books: readonly Book[]) { + const index = books.map((book) => ({ + book, + title: normalized(book.title), + words: normalized(book.title + " " + book.author), + grams: grams(normalized(book.title)), + })); + return (query: string): Book[] => { + const q = normalized(query).slice(0, 120); + if (q.length < 2) return []; + const tokens = q.split(" "), + qgrams = grams(q); + const matches: { book: Book; score: number }[] = []; + for (const entry of index) { + const overlap = [...qgrams].filter((g) => entry.grams.has(g)).length; + const similarity = (2 * overlap) / Math.max(1, qgrams.size + entry.grams.size); + const score = + (entry.title === q ? 10 : entry.title.includes(q) ? 5 : 0) + + (tokens.every((t) => entry.words.includes(t)) ? 2 : 0) + + similarity; + if (score >= 0.24) matches.push({ book: entry.book, score }); + } + return matches + .sort((a, b) => b.score - a.score || a.book.ebook - b.book.ebook) + .slice(0, 20) + .map((entry) => entry.book); + }; +} +export interface Excerpt { + text: string; + words: number; +} +/** The first paragraph boundary at or after roughly 1,000 words. Streaming + * callers wait for that boundary (or EOF), not the rest of the book. */ +export function openingExcerpt(text: string, complete = false): Excerpt | null { + let body = text.replace(/^\uFEFF/, "").replaceAll("\r", ""); + const start = /\*\*\*\s*START OF (?:THE|THIS) PROJECT GUTENBERG EBOOK[^\n]*\n/i.exec(body); + if (start) body = body.slice(start.index + start[0].length); + else if (!complete) return null; + const end = /\*\*\*\s*END OF (?:THE|THIS) PROJECT GUTENBERG EBOOK/i.exec(body); + if (end) { + body = body.slice(0, end.index); + complete = true; + } + const paragraphs = body.split(/\n[ \t]*\n+/); + if (!complete) paragraphs.pop(); + const result: string[] = []; + let words = 0; + for (const paragraph of paragraphs) { + const part = paragraph.trim(); + if (!part) continue; + result.push(part); + words += part.split(/\s+/u).length; + if (words >= 1000) break; + } + // An unusually large single paragraph must still fit the document contract. + // Prefer its last word boundary; ordinary excerpts always keep paragraphs whole. + if (result.join("\n\n").length > 56000 || (body.length >= 56000 && words < 1000)) { + const prefix = body.slice(0, 56000), + at = prefix.lastIndexOf(" "); + const bounded = (at > 0 ? prefix.slice(0, at) : prefix).replace(/[\uD800-\uDBFF]$/, "").trim(); + return { text: bounded, words: bounded ? bounded.split(/\s+/u).length : 0 }; + } + if (words < 1000 && !complete) return null; + return { text: result.join("\n\n"), words }; +} diff --git a/src/download.ts b/src/download.ts new file mode 100644 index 0000000..c81db57 --- /dev/null +++ b/src/download.ts @@ -0,0 +1,82 @@ +import { openingExcerpt, type Excerpt } from "./catalog.js"; +/** User-requested downloads only, from Gutenberg's listed generated-files mirror. + * One fetch at a time, at least two seconds apart; coalesce requests and bound the cache. */ +export function createBookLoader(fetchText: (ebook: number) => Promise, delayMs = 2000) { + const cache = new Map(); + const pending = new Map>(); + let queue = Promise.resolve(), + next = 0, + bytes = 0; + return (ebook: number): Promise => { + const cached = cache.get(ebook); + if (cached) { + cache.delete(ebook); + cache.set(ebook, cached); + return Promise.resolve(cached.excerpt); + } + const existing = pending.get(ebook); + if (existing) return existing; + if (pending.size >= 8) return Promise.reject(Error("Too many book downloads are queued. Try again shortly.")); + const task = queue.then(async () => { + await new Promise((resolve) => setTimeout(resolve, Math.max(0, next - Date.now()))); + next = Date.now() + delayMs; + const text = await fetchText(ebook), + excerpt = openingExcerpt(text, true)!, + size = excerpt.text.length * 2; + while (cache.size && (cache.size >= 8 || bytes + size > 32 * 1024 * 1024)) { + const key = cache.keys().next().value!; + bytes -= cache.get(key)!.bytes; + cache.delete(key); + } + if (size <= 32 * 1024 * 1024) { + cache.set(ebook, { excerpt, bytes: size }); + bytes += size; + } + return excerpt; + }); + queue = task.then( + () => {}, + () => {}, + ); + pending.set(ebook, task); + void task.finally(() => pending.delete(ebook)).catch(() => {}); + return task; + }; +} +export const bookURL = (ebook: number) => `https://gutenberg.pglaf.org/cache/epub/${ebook}/pg${ebook}.txt`; +export async function fetchBook(ebook: number): Promise { + const response = await fetch(bookURL(ebook), { + redirect: "error", + signal: AbortSignal.timeout(20000), + headers: { "User-Agent": "Quixos-Gutenberg-reader/1.0 (on-demand plain-text reader)" }, + }); + if (!response.ok) + throw Error( + response.status === 404 + ? "No UTF-8 plain-text edition is available on this mirror." + : `Book download failed (${response.status}). Try again later.`, + ); + if (!response.body) throw Error("Empty book response"); + const reader = response.body.getReader(), + decoder = new TextDecoder("utf-8", { fatal: true }); + let size = 0, + text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + text += decoder.decode(); + return text; + } + size += value.byteLength; + if (size > 12 * 1024 * 1024) throw Error("This book exceeds the 12 MiB download limit."); + text += decoder.decode(value, { stream: true }); + if (/^\s* ports.catalog.create({ name: "Project Gutenberg" }, { key: "catalog" }), +}; +import { createRegistry } from "./gen/qx.js"; +export const registry = createRegistry(handlers); diff --git a/src/remote.ts b/src/remote.ts new file mode 100644 index 0000000..ead2d91 --- /dev/null +++ b/src/remote.ts @@ -0,0 +1,21 @@ +import { createRegistry } from "./gen/remote.js"; +import books from "./index-data.js"; +import { searchIndex } from "./catalog.js"; +import { createBookLoader, fetchBook } from "./download.js"; +const search = searchIndex(books), + byId = new Map(books.map((book) => [book.ebook, book])); +const load = createBookLoader(fetchBook); +export const registry = createRegistry({ + search: ({ input }) => search(input), + readBook: async ({ input, signal }) => { + const book = byId.get(input.ebook); + if (!book) throw Error("Book is not in this catalog snapshot."); + if (signal.aborted) throw Error("Book request cancelled."); + const excerpt = await load(input.ebook); + return { + book, + url: `https://www.gutenberg.org/ebooks/${book.ebook}`, + ...excerpt, + }; + }, +}); diff --git a/test/catalog.test.mjs b/test/catalog.test.mjs new file mode 100644 index 0000000..56d9c6e --- /dev/null +++ b/test/catalog.test.mjs @@ -0,0 +1,128 @@ +import assert from "node:assert/strict"; +import { test, after } from "node:test"; +import { stripTypeScriptTypes } from "node:module"; +import { readFile, mkdtemp, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +const directory = await mkdtemp(join(tmpdir(), "gutenberg-test-")); +for (const name of ["catalog", "download"]) { + const source = await readFile(new URL("../src/" + name + ".ts", import.meta.url), "utf8"); + await writeFile( + join(directory, name + ".mjs"), + stripTypeScriptTypes(source).replace("./catalog.js", "./catalog.mjs"), + ); +} +after(() => rm(directory, { recursive: true, force: true })); +const { parseIndex, searchIndex, openingExcerpt } = await import(pathToFileURL(join(directory, "catalog.mjs"))); +const { createBookLoader, bookURL, fetchBook } = await import(pathToFileURL(join(directory, "download.mjs"))); +test("CSV preserves quoted/multiline titles and selects text editions", () => { + const books = parseIndex( + 'Text#,Type,Title,Language,Authors\r\n1342,Text,"Pride, and\r\nPrejudice",en,"Austen, Jane"\r\n11,Text,"Alice and ""friends""",en,Carroll\r\n12,Sound,Audio,en,Someone\r\n', + ); + assert.deepEqual(books, [ + { ebook: 1342, title: "Pride, and Prejudice", language: "en", author: "Austen, Jane" }, + { ebook: 11, title: 'Alice and "friends"', language: "en", author: "Carroll" }, + ]); + assert.throws(() => parseIndex('Text#,Type,Title,Language,Authors\n1,Text,"broken'), /Unclosed/); +}); +test("fuzzy titles and diacritics work, with a bounded result list", () => { + const books = [ + { ebook: 1342, title: "Pride and Prejudice", author: "Austen, Jane", language: "en" }, + { ebook: 11, title: "Alice in Wonderland", author: "Carroll", language: "en" }, + ...Array.from({ length: 40 }, (_, i) => ({ + ebook: 100 + i, + title: "Émile " + i, + author: "Rousseau", + language: "fr", + })), + ]; + const search = searchIndex(books); + assert.equal(search("prde and prejudi").at(0).ebook, 1342); + assert.equal(search("austen").at(0).ebook, 1342); + assert.equal(search("emile").length, 20); + assert.deepEqual(search("a"), []); +}); +test("opening excerpt keeps paragraph boundaries and excludes Gutenberg framing", () => { + const body = + "license\n*** START OF THE PROJECT GUTENBERG EBOOK TEST ***\n\nFirst\nline\n\nSecond\n\n*** END OF THE PROJECT GUTENBERG EBOOK TEST ***\nlicense"; + assert.deepEqual(openingExcerpt(body), { text: "First\nline\n\nSecond", words: 3 }); + const first = "word ".repeat(600).trim(), + second = "more ".repeat(450).trim(); + const text = "*** START OF THE PROJECT GUTENBERG EBOOK TEST ***\n\n" + first + "\n\n" + second; + assert.equal(openingExcerpt(text), null, "Wait for the whole final paragraph"); + assert.deepEqual(openingExcerpt(text + "\n\nignored"), { text: first + "\n\n" + second, words: 1050 }); + assert.deepEqual(openingExcerpt("short book", true), { text: "short book", words: 2 }); + const long = "😀".repeat(30000); + const bounded = openingExcerpt(long, true); + assert(bounded.text.length <= 56000); + assert(!/[\uD800-\uDBFF]$/.test(bounded.text)); +}); +test("streaming fetch stops at the excerpt and releases the remaining download", async () => { + const original = globalThis.fetch; + let cancelled = false, + reads = 0; + const chunks = [ + "licence\n*** START OF THE PROJECT GUTEN", + "BERG EBOOK TEST ***\n\n", + "word ".repeat(600) + "\n\n", + "more ".repeat(450) + "\n\n", + "unused rest of the book", + ]; + globalThis.fetch = async () => + new Response( + new ReadableStream( + { + pull(controller) { + reads++; + const chunk = chunks.shift(); + if (chunk) controller.enqueue(new TextEncoder().encode(chunk)); + else controller.close(); + }, + cancel() { + cancelled = true; + }, + }, + { highWaterMark: 0 }, + ), + ); + try { + const text = await fetchBook(1342); + assert.equal(reads, 4); + assert.equal(cancelled, true); + assert.equal(openingExcerpt(text, true).words, 1050); + assert(!text.includes("unused")); + } finally { + globalThis.fetch = original; + } +}); +test("downloads coalesce, cache, serialize, and permit retries after failure", async () => { + let calls = 0, + active = 0, + maximum = 0, + fail = true; + const load = createBookLoader(async (ebook) => { + calls++; + active++; + maximum = Math.max(maximum, active); + await new Promise((r) => setTimeout(r, 2)); + active--; + if (ebook === 3 && fail) { + fail = false; + throw Error("offline"); + } + return "Book " + ebook; + }, 1); + assert.deepEqual(await Promise.all([load(1), load(1), load(2)]), [ + { text: "Book 1", words: 2 }, + { text: "Book 1", words: 2 }, + { text: "Book 2", words: 2 }, + ]); + assert.equal(calls, 2); + assert.equal(maximum, 1); + await load(1); + assert.equal(calls, 2); + await assert.rejects(load(3), /offline/); + assert.deepEqual(await load(3), { text: "Book 3", words: 2 }); + assert.equal(bookURL(1342), "https://gutenberg.pglaf.org/cache/epub/1342/pg1342.txt"); +});