Export template source from 11bc3ae35b0034c113daca4c35d1b341aa31ef70:examples/local-first/packages/gutenberg-runtime
This commit is contained in:
Quixos Template Publisher
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"monorepoCommit": "11bc3ae35b0034c113daca4c35d1b341aa31ef70",
|
||||
"sourcePath": "examples/local-first/packages/gutenberg-runtime"
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
# Project Gutenberg source for Reader
|
||||
|
||||
Updated September 30, 2026. The catalog is an ordinary Camino entity with
|
||||
`search` and `readBook` remote operations. It has an independent Studio
|
||||
presentation and an embeddable `GutenbergPicker`. Reader's document-author
|
||||
adapter imports the chosen excerpt through the existing durable publication
|
||||
connection; completion navigates the bookshelf to the new document. Manual
|
||||
text authoring keeps its own independent state.
|
||||
|
||||
## Catalog and downloads
|
||||
|
||||
`catalog.csv.gz` contains the reviewed September 30, 2026 snapshot of
|
||||
Gutenberg's machine-readable CSV. Its SHA-256 is
|
||||
`62a53fdce11252795b8f247d4cdb51d100a2cf4856a716ff59d53959ad1c19c3`.
|
||||
The build parses text editions and bundles the index into the service. Fuzzy
|
||||
search over titles/authors reads that index without contacting Gutenberg.
|
||||
Replace the snapshot deliberately and repin workspace resources when refreshing
|
||||
it. The daily feed is not a build-time dependency: older source revisions must
|
||||
remain buildable even after it changes. The index builder checks coverage and
|
||||
representative titles before accepting a snapshot.
|
||||
|
||||
Book text is requested when a user opens a result, from the
|
||||
[listed generated-files mirror](https://www.gutenberg.org/dirs/MIRRORS.ALL)
|
||||
`gutenberg.pglaf.org`, following the
|
||||
[robot access policy](https://www.gutenberg.org/policy/robot_access.html).
|
||||
The downloader reads only far enough to collect about 1,000 words, ending at
|
||||
the next complete paragraph, then cancels the remaining response body. Gutenberg
|
||||
header/footer markers are removed. Short books end at EOF. Exceptionally long
|
||||
paragraphs are bounded at 56,000 characters, preferring a word boundary and
|
||||
preserving Unicode surrogate pairs.
|
||||
|
||||
`readBook({ebook})` returns `{book, url, text, words}` in one response. The picker
|
||||
shows a preview and an **Import excerpt** button; the standalone catalog offers
|
||||
**Copy excerpt**. There is no paragraph pagination or selection step.
|
||||
|
||||
Downloads are serialized, start at least two seconds apart, and coalesce requests
|
||||
for the same book. The process caches eight excerpts, bounded by 32 MiB. The UI
|
||||
also keeps eight excerpts while mounted, so going back to a book needs no new
|
||||
remote call. The first request still needs the mirror to respond. Downloads have
|
||||
a 20-second timeout and 12 MiB maximum; missing editions and failures can be
|
||||
retried. Search never downloads books.
|
||||
|
||||
## Verification
|
||||
|
||||
```sh
|
||||
node --test examples/local-first/packages/gutenberg-runtime/test/catalog.test.mjs
|
||||
nix run .#camino-presentation-qualification
|
||||
```
|
||||
|
||||
Unit tests cover paragraph/Unicode boundaries, truncated streams, stopping and
|
||||
cancelling a download, coalescing, caching, serialization and retries. Browser
|
||||
checks search the actual pinned index and stub only the external book response.
|
||||
@@ -0,0 +1,11 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { parseIndex } from "./src/catalog.ts";
|
||||
const books = parseIndex(await readFile(process.argv[2], "utf8"));
|
||||
if (
|
||||
books.length < 70000 ||
|
||||
!books.some((b) => b.ebook === 1342 && b.title.includes("Pride")) ||
|
||||
!books.some((b) => b.ebook === 11 && b.title.includes("Alice"))
|
||||
)
|
||||
throw Error("Gutenberg CSV format/coverage changed; review parser before updating pin");
|
||||
await writeFile(process.argv[3], "export default " + JSON.stringify(books) + ";\n");
|
||||
console.log("Indexed", books.length, "Gutenberg titles");
|
||||
Binary file not shown.
Generated
+108
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"nodes": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"helpers": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1790852599,
|
||||
"narHash": "sha256-MdNtZpcsoguLPSCpAy9Naz6fkVPrftbZQbz1Aj7Zm9A=",
|
||||
"ref": "refs/tags/quixos-reachability/91679e8cc82133093cb0b4f9e519cbb39a5bba4f",
|
||||
"rev": "91679e8cc82133093cb0b4f9e519cbb39a5bba4f",
|
||||
"revCount": 54,
|
||||
"type": "git",
|
||||
"url": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git"
|
||||
},
|
||||
"original": {
|
||||
"ref": "refs/tags/quixos-reachability/91679e8cc82133093cb0b4f9e519cbb39a5bba4f",
|
||||
"rev": "91679e8cc82133093cb0b4f9e519cbb39a5bba4f",
|
||||
"type": "git",
|
||||
"url": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1783776592,
|
||||
"narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"protocol": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1791261504,
|
||||
"narHash": "sha256-QSk9DWR7n5KL6cMHzmSKKXNIyaFA0s8Nu+wXjhhlENI=",
|
||||
"ref": "refs/tags/quixos-reachability/5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87",
|
||||
"rev": "5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87",
|
||||
"revCount": 184,
|
||||
"type": "git",
|
||||
"url": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-protocol.git"
|
||||
},
|
||||
"original": {
|
||||
"ref": "refs/tags/quixos-reachability/5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87",
|
||||
"rev": "5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87",
|
||||
"type": "git",
|
||||
"url": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-protocol.git"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": [
|
||||
"protocol",
|
||||
"flake-utils"
|
||||
],
|
||||
"helpers": "helpers",
|
||||
"nixpkgs": [
|
||||
"protocol",
|
||||
"nixpkgs"
|
||||
],
|
||||
"protocol": "protocol"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
inputs.protocol.url = "git+https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-protocol.git?ref=refs/tags/quixos-reachability/5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87&rev=5b44c8154dc038efd87cf1cbdd2c00b66bfd4f87";
|
||||
inputs.nixpkgs.follows = "protocol/nixpkgs";
|
||||
inputs.flake-utils.follows = "protocol/flake-utils";
|
||||
inputs.helpers = {
|
||||
url = "git+https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git?ref=refs/tags/quixos-reachability/91679e8cc82133093cb0b4f9e519cbb39a5bba4f&rev=91679e8cc82133093cb0b4f9e519cbb39a5bba4f";
|
||||
flake = false;
|
||||
};
|
||||
outputs =
|
||||
{
|
||||
nixpkgs,
|
||||
flake-utils,
|
||||
helpers,
|
||||
...
|
||||
}:
|
||||
flake-utils.lib.eachDefaultSystem (
|
||||
system:
|
||||
let
|
||||
pkgs = import nixpkgs { inherit system; };
|
||||
camino = import (helpers + "/quixos-package-helpers.nix");
|
||||
in
|
||||
{
|
||||
quixosPackages.checkedArtifacts =
|
||||
context:
|
||||
let
|
||||
pkgs = camino.packageSet { inherit nixpkgs system context; };
|
||||
# Gutenberg regenerates its feed daily. Keep the reviewed bytes in
|
||||
# the source revision so a clean machine can still build old pins.
|
||||
catalog = ./catalog.csv.gz;
|
||||
source =
|
||||
pkgs.runCommand "gutenberg-source"
|
||||
{
|
||||
nativeBuildInputs = [
|
||||
pkgs.nodejs_24
|
||||
pkgs.gzip
|
||||
];
|
||||
}
|
||||
''
|
||||
cp -r ${./.} "$out"
|
||||
chmod -R u+w "$out"
|
||||
gzip -dc ${catalog} > catalog.csv
|
||||
node "$out/build-index.mjs" catalog.csv "$out/src/index-data.ts"
|
||||
'';
|
||||
portable = camino.mkCaminoTypeScriptPackage {
|
||||
inherit pkgs context;
|
||||
src = source;
|
||||
entry = "src/portable.ts";
|
||||
};
|
||||
remote = camino.mkCaminoTypeScriptService {
|
||||
inherit pkgs context;
|
||||
src = source;
|
||||
entry = "src/remote.ts";
|
||||
};
|
||||
in
|
||||
camino.assembleCaminoPackage {
|
||||
inherit pkgs context;
|
||||
artifacts = [
|
||||
portable
|
||||
remote
|
||||
];
|
||||
};
|
||||
devShells.default = camino.mkCaminoYarnShell { inherit pkgs; };
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"name": "@quixos-example/gutenberg-runtime",
|
||||
"private": true,
|
||||
"type": "module"
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
symbol JavaScript id "target:org.quixos.execution.javascript@1";
|
||||
external atom GutenbergCatalogObject id "atom:gutenberg:catalog";
|
||||
import interface GutenbergCatalog;
|
||||
type GutenbergBook = record {ebook:uint32;title:string;author:string;language:string;};
|
||||
type GutenbergExcerpt = record {book:GutenbergBook;url:string;text:string;words:uint32;};
|
||||
package GutenbergRuntime id "package:gutenberg:runtime" revision "package:gutenberg:runtime@1" {
|
||||
function create id "gutenberg:create" : unit -> atom-ref<GutenbergCatalogObject> execution transaction targets [JavaScript,server] requires {creation catalog id "catalog" : GutenbergCatalogObject input record {name:string;};};
|
||||
operation search id "gutenberg:search" : string -> list<GutenbergBook> mode call execution remote receiver any;
|
||||
operation readBook id "gutenberg:read" : record {ebook:uint32;} -> GutenbergExcerpt mode call execution remote receiver any;
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
quixos-lock version 1 {
|
||||
quixos source { repository "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos.git"; commit "11bc3ae35b0034c113daca4c35d1b341aa31ef70"; }
|
||||
interface GutenbergCatalog source { repository "https://repos.quixos.org/quixos-local-first-examples/interface-gutenberg-catalog.git"; commit "af88f1abc529f82f4464e4c0ed99cd2fff6b82ea"; }
|
||||
}
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
export interface Book {
|
||||
ebook: number;
|
||||
title: string;
|
||||
author: string;
|
||||
language: string;
|
||||
}
|
||||
/** RFC 4180 records, including quoted commas, escaped quotes, and multiline titles. */
|
||||
export function parseIndex(text: string): Book[] {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [],
|
||||
field = "",
|
||||
quoted = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const ch = text[i]!;
|
||||
if (ch === '"') {
|
||||
if (quoted && text[i + 1] === '"') {
|
||||
field += '"';
|
||||
i++;
|
||||
} else if (quoted || field === "") quoted = !quoted;
|
||||
else throw Error("Unexpected CSV quote");
|
||||
} else if (!quoted && (ch === "," || ch === "\n" || ch === "\r")) {
|
||||
row.push(field);
|
||||
field = "";
|
||||
if (ch !== ",") {
|
||||
rows.push(row);
|
||||
row = [];
|
||||
if (ch === "\r" && text[i + 1] === "\n") i++;
|
||||
}
|
||||
} else field += ch;
|
||||
}
|
||||
if (quoted) throw Error("Unclosed CSV quote");
|
||||
if (field || row.length) {
|
||||
row.push(field);
|
||||
rows.push(row);
|
||||
}
|
||||
const header = rows.shift()!;
|
||||
const columns = ["Text#", "Type", "Title", "Language", "Authors"].map((name) => header.indexOf(name));
|
||||
if (columns.some((i) => i < 0)) throw Error("Unrecognized Gutenberg CSV header");
|
||||
const books: Book[] = [],
|
||||
seen = new Set<number>();
|
||||
for (const row of rows) {
|
||||
if (row.length === 1 && row[0] === "") continue;
|
||||
if (row.length !== header.length) throw Error("Invalid Gutenberg CSV row");
|
||||
if (row[columns[1]!] !== "Text") continue;
|
||||
const ebook = Number(row[columns[0]!]!);
|
||||
if (!Number.isSafeInteger(ebook) || ebook < 1 || seen.has(ebook)) throw Error("Invalid or repeated book ID");
|
||||
seen.add(ebook);
|
||||
books.push({
|
||||
ebook,
|
||||
title: row[columns[2]!]!.replace(/\s+/g, " ").trim(),
|
||||
language: row[columns[3]!]!,
|
||||
author: row[columns[4]!]!,
|
||||
});
|
||||
}
|
||||
return books;
|
||||
}
|
||||
const normalized = (text: string) =>
|
||||
text
|
||||
.normalize("NFKD")
|
||||
.replace(/\p{M}/gu, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{L}\p{N}]+/gu, " ")
|
||||
.trim();
|
||||
const grams = (text: string) =>
|
||||
new Set(Array.from({ length: Math.max(0, text.length - 2) }, (_, i) => text.slice(i, i + 3)));
|
||||
export function searchIndex(books: readonly Book[]) {
|
||||
const index = books.map((book) => ({
|
||||
book,
|
||||
title: normalized(book.title),
|
||||
words: normalized(book.title + " " + book.author),
|
||||
grams: grams(normalized(book.title)),
|
||||
}));
|
||||
return (query: string): Book[] => {
|
||||
const q = normalized(query).slice(0, 120);
|
||||
if (q.length < 2) return [];
|
||||
const tokens = q.split(" "),
|
||||
qgrams = grams(q);
|
||||
const matches: { book: Book; score: number }[] = [];
|
||||
for (const entry of index) {
|
||||
const overlap = [...qgrams].filter((g) => entry.grams.has(g)).length;
|
||||
const similarity = (2 * overlap) / Math.max(1, qgrams.size + entry.grams.size);
|
||||
const score =
|
||||
(entry.title === q ? 10 : entry.title.includes(q) ? 5 : 0) +
|
||||
(tokens.every((t) => entry.words.includes(t)) ? 2 : 0) +
|
||||
similarity;
|
||||
if (score >= 0.24) matches.push({ book: entry.book, score });
|
||||
}
|
||||
return matches
|
||||
.sort((a, b) => b.score - a.score || a.book.ebook - b.book.ebook)
|
||||
.slice(0, 20)
|
||||
.map((entry) => entry.book);
|
||||
};
|
||||
}
|
||||
export interface Excerpt {
|
||||
text: string;
|
||||
words: number;
|
||||
}
|
||||
/** The first paragraph boundary at or after roughly 1,000 words. Streaming
|
||||
* callers wait for that boundary (or EOF), not the rest of the book. */
|
||||
export function openingExcerpt(text: string, complete = false): Excerpt | null {
|
||||
let body = text.replace(/^\uFEFF/, "").replaceAll("\r", "");
|
||||
const start = /\*\*\*\s*START OF (?:THE|THIS) PROJECT GUTENBERG EBOOK[^\n]*\n/i.exec(body);
|
||||
if (start) body = body.slice(start.index + start[0].length);
|
||||
else if (!complete) return null;
|
||||
const end = /\*\*\*\s*END OF (?:THE|THIS) PROJECT GUTENBERG EBOOK/i.exec(body);
|
||||
if (end) {
|
||||
body = body.slice(0, end.index);
|
||||
complete = true;
|
||||
}
|
||||
const paragraphs = body.split(/\n[ \t]*\n+/);
|
||||
if (!complete) paragraphs.pop();
|
||||
const result: string[] = [];
|
||||
let words = 0;
|
||||
for (const paragraph of paragraphs) {
|
||||
const part = paragraph.trim();
|
||||
if (!part) continue;
|
||||
result.push(part);
|
||||
words += part.split(/\s+/u).length;
|
||||
if (words >= 1000) break;
|
||||
}
|
||||
// An unusually large single paragraph must still fit the document contract.
|
||||
// Prefer its last word boundary; ordinary excerpts always keep paragraphs whole.
|
||||
if (result.join("\n\n").length > 56000 || (body.length >= 56000 && words < 1000)) {
|
||||
const prefix = body.slice(0, 56000),
|
||||
at = prefix.lastIndexOf(" ");
|
||||
const bounded = (at > 0 ? prefix.slice(0, at) : prefix).replace(/[\uD800-\uDBFF]$/, "").trim();
|
||||
return { text: bounded, words: bounded ? bounded.split(/\s+/u).length : 0 };
|
||||
}
|
||||
if (words < 1000 && !complete) return null;
|
||||
return { text: result.join("\n\n"), words };
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import { openingExcerpt, type Excerpt } from "./catalog.js";
|
||||
/** User-requested downloads only, from Gutenberg's listed generated-files mirror.
|
||||
* One fetch at a time, at least two seconds apart; coalesce requests and bound the cache. */
|
||||
export function createBookLoader(fetchText: (ebook: number) => Promise<string>, delayMs = 2000) {
|
||||
const cache = new Map<number, { excerpt: Excerpt; bytes: number }>();
|
||||
const pending = new Map<number, Promise<Excerpt>>();
|
||||
let queue = Promise.resolve(),
|
||||
next = 0,
|
||||
bytes = 0;
|
||||
return (ebook: number): Promise<Excerpt> => {
|
||||
const cached = cache.get(ebook);
|
||||
if (cached) {
|
||||
cache.delete(ebook);
|
||||
cache.set(ebook, cached);
|
||||
return Promise.resolve(cached.excerpt);
|
||||
}
|
||||
const existing = pending.get(ebook);
|
||||
if (existing) return existing;
|
||||
if (pending.size >= 8) return Promise.reject(Error("Too many book downloads are queued. Try again shortly."));
|
||||
const task = queue.then(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.max(0, next - Date.now())));
|
||||
next = Date.now() + delayMs;
|
||||
const text = await fetchText(ebook),
|
||||
excerpt = openingExcerpt(text, true)!,
|
||||
size = excerpt.text.length * 2;
|
||||
while (cache.size && (cache.size >= 8 || bytes + size > 32 * 1024 * 1024)) {
|
||||
const key = cache.keys().next().value!;
|
||||
bytes -= cache.get(key)!.bytes;
|
||||
cache.delete(key);
|
||||
}
|
||||
if (size <= 32 * 1024 * 1024) {
|
||||
cache.set(ebook, { excerpt, bytes: size });
|
||||
bytes += size;
|
||||
}
|
||||
return excerpt;
|
||||
});
|
||||
queue = task.then(
|
||||
() => {},
|
||||
() => {},
|
||||
);
|
||||
pending.set(ebook, task);
|
||||
void task.finally(() => pending.delete(ebook)).catch(() => {});
|
||||
return task;
|
||||
};
|
||||
}
|
||||
export const bookURL = (ebook: number) => `https://gutenberg.pglaf.org/cache/epub/${ebook}/pg${ebook}.txt`;
|
||||
export async function fetchBook(ebook: number): Promise<string> {
|
||||
const response = await fetch(bookURL(ebook), {
|
||||
redirect: "error",
|
||||
signal: AbortSignal.timeout(20000),
|
||||
headers: { "User-Agent": "Quixos-Gutenberg-reader/1.0 (on-demand plain-text reader)" },
|
||||
});
|
||||
if (!response.ok)
|
||||
throw Error(
|
||||
response.status === 404
|
||||
? "No UTF-8 plain-text edition is available on this mirror."
|
||||
: `Book download failed (${response.status}). Try again later.`,
|
||||
);
|
||||
if (!response.body) throw Error("Empty book response");
|
||||
const reader = response.body.getReader(),
|
||||
decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
let size = 0,
|
||||
text = "";
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
text += decoder.decode();
|
||||
return text;
|
||||
}
|
||||
size += value.byteLength;
|
||||
if (size > 12 * 1024 * 1024) throw Error("This book exceeds the 12 MiB download limit.");
|
||||
text += decoder.decode(value, { stream: true });
|
||||
if (/^\s*<!doctype html|^\s*<html/i.test(text))
|
||||
throw Error("The mirror returned an HTML page instead of book text.");
|
||||
if (openingExcerpt(text)) return text;
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel();
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import type { PortableImplementation } from "./gen/qx.js";
|
||||
export const handlers: PortableImplementation = {
|
||||
create: ({ ports }) => ports.catalog.create({ name: "Project Gutenberg" }, { key: "catalog" }),
|
||||
};
|
||||
import { createRegistry } from "./gen/qx.js";
|
||||
export const registry = createRegistry(handlers);
|
||||
@@ -0,0 +1,21 @@
|
||||
import { createRegistry } from "./gen/remote.js";
|
||||
import books from "./index-data.js";
|
||||
import { searchIndex } from "./catalog.js";
|
||||
import { createBookLoader, fetchBook } from "./download.js";
|
||||
const search = searchIndex(books),
|
||||
byId = new Map(books.map((book) => [book.ebook, book]));
|
||||
const load = createBookLoader(fetchBook);
|
||||
export const registry = createRegistry({
|
||||
search: ({ input }) => search(input),
|
||||
readBook: async ({ input, signal }) => {
|
||||
const book = byId.get(input.ebook);
|
||||
if (!book) throw Error("Book is not in this catalog snapshot.");
|
||||
if (signal.aborted) throw Error("Book request cancelled.");
|
||||
const excerpt = await load(input.ebook);
|
||||
return {
|
||||
book,
|
||||
url: `https://www.gutenberg.org/ebooks/${book.ebook}`,
|
||||
...excerpt,
|
||||
};
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,128 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test, after } from "node:test";
|
||||
import { stripTypeScriptTypes } from "node:module";
|
||||
import { readFile, mkdtemp, writeFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
const directory = await mkdtemp(join(tmpdir(), "gutenberg-test-"));
|
||||
for (const name of ["catalog", "download"]) {
|
||||
const source = await readFile(new URL("../src/" + name + ".ts", import.meta.url), "utf8");
|
||||
await writeFile(
|
||||
join(directory, name + ".mjs"),
|
||||
stripTypeScriptTypes(source).replace("./catalog.js", "./catalog.mjs"),
|
||||
);
|
||||
}
|
||||
after(() => rm(directory, { recursive: true, force: true }));
|
||||
const { parseIndex, searchIndex, openingExcerpt } = await import(pathToFileURL(join(directory, "catalog.mjs")));
|
||||
const { createBookLoader, bookURL, fetchBook } = await import(pathToFileURL(join(directory, "download.mjs")));
|
||||
test("CSV preserves quoted/multiline titles and selects text editions", () => {
|
||||
const books = parseIndex(
|
||||
'Text#,Type,Title,Language,Authors\r\n1342,Text,"Pride, and\r\nPrejudice",en,"Austen, Jane"\r\n11,Text,"Alice and ""friends""",en,Carroll\r\n12,Sound,Audio,en,Someone\r\n',
|
||||
);
|
||||
assert.deepEqual(books, [
|
||||
{ ebook: 1342, title: "Pride, and Prejudice", language: "en", author: "Austen, Jane" },
|
||||
{ ebook: 11, title: 'Alice and "friends"', language: "en", author: "Carroll" },
|
||||
]);
|
||||
assert.throws(() => parseIndex('Text#,Type,Title,Language,Authors\n1,Text,"broken'), /Unclosed/);
|
||||
});
|
||||
test("fuzzy titles and diacritics work, with a bounded result list", () => {
|
||||
const books = [
|
||||
{ ebook: 1342, title: "Pride and Prejudice", author: "Austen, Jane", language: "en" },
|
||||
{ ebook: 11, title: "Alice in Wonderland", author: "Carroll", language: "en" },
|
||||
...Array.from({ length: 40 }, (_, i) => ({
|
||||
ebook: 100 + i,
|
||||
title: "Émile " + i,
|
||||
author: "Rousseau",
|
||||
language: "fr",
|
||||
})),
|
||||
];
|
||||
const search = searchIndex(books);
|
||||
assert.equal(search("prde and prejudi").at(0).ebook, 1342);
|
||||
assert.equal(search("austen").at(0).ebook, 1342);
|
||||
assert.equal(search("emile").length, 20);
|
||||
assert.deepEqual(search("a"), []);
|
||||
});
|
||||
test("opening excerpt keeps paragraph boundaries and excludes Gutenberg framing", () => {
|
||||
const body =
|
||||
"license\n*** START OF THE PROJECT GUTENBERG EBOOK TEST ***\n\nFirst\nline\n\nSecond\n\n*** END OF THE PROJECT GUTENBERG EBOOK TEST ***\nlicense";
|
||||
assert.deepEqual(openingExcerpt(body), { text: "First\nline\n\nSecond", words: 3 });
|
||||
const first = "word ".repeat(600).trim(),
|
||||
second = "more ".repeat(450).trim();
|
||||
const text = "*** START OF THE PROJECT GUTENBERG EBOOK TEST ***\n\n" + first + "\n\n" + second;
|
||||
assert.equal(openingExcerpt(text), null, "Wait for the whole final paragraph");
|
||||
assert.deepEqual(openingExcerpt(text + "\n\nignored"), { text: first + "\n\n" + second, words: 1050 });
|
||||
assert.deepEqual(openingExcerpt("short book", true), { text: "short book", words: 2 });
|
||||
const long = "😀".repeat(30000);
|
||||
const bounded = openingExcerpt(long, true);
|
||||
assert(bounded.text.length <= 56000);
|
||||
assert(!/[\uD800-\uDBFF]$/.test(bounded.text));
|
||||
});
|
||||
test("streaming fetch stops at the excerpt and releases the remaining download", async () => {
|
||||
const original = globalThis.fetch;
|
||||
let cancelled = false,
|
||||
reads = 0;
|
||||
const chunks = [
|
||||
"licence\n*** START OF THE PROJECT GUTEN",
|
||||
"BERG EBOOK TEST ***\n\n",
|
||||
"word ".repeat(600) + "\n\n",
|
||||
"more ".repeat(450) + "\n\n",
|
||||
"unused rest of the book",
|
||||
];
|
||||
globalThis.fetch = async () =>
|
||||
new Response(
|
||||
new ReadableStream(
|
||||
{
|
||||
pull(controller) {
|
||||
reads++;
|
||||
const chunk = chunks.shift();
|
||||
if (chunk) controller.enqueue(new TextEncoder().encode(chunk));
|
||||
else controller.close();
|
||||
},
|
||||
cancel() {
|
||||
cancelled = true;
|
||||
},
|
||||
},
|
||||
{ highWaterMark: 0 },
|
||||
),
|
||||
);
|
||||
try {
|
||||
const text = await fetchBook(1342);
|
||||
assert.equal(reads, 4);
|
||||
assert.equal(cancelled, true);
|
||||
assert.equal(openingExcerpt(text, true).words, 1050);
|
||||
assert(!text.includes("unused"));
|
||||
} finally {
|
||||
globalThis.fetch = original;
|
||||
}
|
||||
});
|
||||
test("downloads coalesce, cache, serialize, and permit retries after failure", async () => {
|
||||
let calls = 0,
|
||||
active = 0,
|
||||
maximum = 0,
|
||||
fail = true;
|
||||
const load = createBookLoader(async (ebook) => {
|
||||
calls++;
|
||||
active++;
|
||||
maximum = Math.max(maximum, active);
|
||||
await new Promise((r) => setTimeout(r, 2));
|
||||
active--;
|
||||
if (ebook === 3 && fail) {
|
||||
fail = false;
|
||||
throw Error("offline");
|
||||
}
|
||||
return "Book " + ebook;
|
||||
}, 1);
|
||||
assert.deepEqual(await Promise.all([load(1), load(1), load(2)]), [
|
||||
{ text: "Book 1", words: 2 },
|
||||
{ text: "Book 1", words: 2 },
|
||||
{ text: "Book 2", words: 2 },
|
||||
]);
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(maximum, 1);
|
||||
await load(1);
|
||||
assert.equal(calls, 2);
|
||||
await assert.rejects(load(3), /offline/);
|
||||
assert.deepEqual(await load(3), { text: "Book 3", words: 2 });
|
||||
assert.equal(bookURL(1342), "https://gutenberg.pglaf.org/cache/epub/1342/pg1342.txt");
|
||||
});
|
||||
Reference in New Issue
Block a user