Export template source from 488f7e229afed8f03fc3bfa4abe707da0c613460:examples/local-first/packages/gutenberg-runtime
This commit is contained in:
Quixos Template Publisher
+131
@@ -0,0 +1,131 @@
|
||||
export interface Book {
|
||||
ebook: number;
|
||||
title: string;
|
||||
author: string;
|
||||
language: string;
|
||||
}
|
||||
/** RFC 4180 records, including quoted commas, escaped quotes, and multiline titles. */
|
||||
export function parseIndex(text: string): Book[] {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [],
|
||||
field = "",
|
||||
quoted = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const ch = text[i]!;
|
||||
if (ch === '"') {
|
||||
if (quoted && text[i + 1] === '"') {
|
||||
field += '"';
|
||||
i++;
|
||||
} else if (quoted || field === "") quoted = !quoted;
|
||||
else throw Error("Unexpected CSV quote");
|
||||
} else if (!quoted && (ch === "," || ch === "\n" || ch === "\r")) {
|
||||
row.push(field);
|
||||
field = "";
|
||||
if (ch !== ",") {
|
||||
rows.push(row);
|
||||
row = [];
|
||||
if (ch === "\r" && text[i + 1] === "\n") i++;
|
||||
}
|
||||
} else field += ch;
|
||||
}
|
||||
if (quoted) throw Error("Unclosed CSV quote");
|
||||
if (field || row.length) {
|
||||
row.push(field);
|
||||
rows.push(row);
|
||||
}
|
||||
const header = rows.shift()!;
|
||||
const columns = ["Text#", "Type", "Title", "Language", "Authors"].map((name) => header.indexOf(name));
|
||||
if (columns.some((i) => i < 0)) throw Error("Unrecognized Gutenberg CSV header");
|
||||
const books: Book[] = [],
|
||||
seen = new Set<number>();
|
||||
for (const row of rows) {
|
||||
if (row.length === 1 && row[0] === "") continue;
|
||||
if (row.length !== header.length) throw Error("Invalid Gutenberg CSV row");
|
||||
if (row[columns[1]!] !== "Text") continue;
|
||||
const ebook = Number(row[columns[0]!]!);
|
||||
if (!Number.isSafeInteger(ebook) || ebook < 1 || seen.has(ebook)) throw Error("Invalid or repeated book ID");
|
||||
seen.add(ebook);
|
||||
books.push({
|
||||
ebook,
|
||||
title: row[columns[2]!]!.replace(/\s+/g, " ").trim(),
|
||||
language: row[columns[3]!]!,
|
||||
author: row[columns[4]!]!,
|
||||
});
|
||||
}
|
||||
return books;
|
||||
}
|
||||
const normalized = (text: string) =>
|
||||
text
|
||||
.normalize("NFKD")
|
||||
.replace(/\p{M}/gu, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{L}\p{N}]+/gu, " ")
|
||||
.trim();
|
||||
const grams = (text: string) =>
|
||||
new Set(Array.from({ length: Math.max(0, text.length - 2) }, (_, i) => text.slice(i, i + 3)));
|
||||
export function searchIndex(books: readonly Book[]) {
|
||||
const index = books.map((book) => ({
|
||||
book,
|
||||
title: normalized(book.title),
|
||||
words: normalized(book.title + " " + book.author),
|
||||
grams: grams(normalized(book.title)),
|
||||
}));
|
||||
return (query: string): Book[] => {
|
||||
const q = normalized(query).slice(0, 120);
|
||||
if (q.length < 2) return [];
|
||||
const tokens = q.split(" "),
|
||||
qgrams = grams(q);
|
||||
const matches: { book: Book; score: number }[] = [];
|
||||
for (const entry of index) {
|
||||
const overlap = [...qgrams].filter((g) => entry.grams.has(g)).length;
|
||||
const similarity = (2 * overlap) / Math.max(1, qgrams.size + entry.grams.size);
|
||||
const score =
|
||||
(entry.title === q ? 10 : entry.title.includes(q) ? 5 : 0) +
|
||||
(tokens.every((t) => entry.words.includes(t)) ? 2 : 0) +
|
||||
similarity;
|
||||
if (score >= 0.24) matches.push({ book: entry.book, score });
|
||||
}
|
||||
return matches
|
||||
.sort((a, b) => b.score - a.score || a.book.ebook - b.book.ebook)
|
||||
.slice(0, 20)
|
||||
.map((entry) => entry.book);
|
||||
};
|
||||
}
|
||||
export interface Excerpt {
|
||||
text: string;
|
||||
words: number;
|
||||
}
|
||||
/** The first paragraph boundary at or after roughly 1,000 words. Streaming
|
||||
* callers wait for that boundary (or EOF), not the rest of the book. */
|
||||
export function openingExcerpt(text: string, complete = false): Excerpt | null {
|
||||
let body = text.replace(/^\uFEFF/, "").replaceAll("\r", "");
|
||||
const start = /\*\*\*\s*START OF (?:THE|THIS) PROJECT GUTENBERG EBOOK[^\n]*\n/i.exec(body);
|
||||
if (start) body = body.slice(start.index + start[0].length);
|
||||
else if (!complete) return null;
|
||||
const end = /\*\*\*\s*END OF (?:THE|THIS) PROJECT GUTENBERG EBOOK/i.exec(body);
|
||||
if (end) {
|
||||
body = body.slice(0, end.index);
|
||||
complete = true;
|
||||
}
|
||||
const paragraphs = body.split(/\n[ \t]*\n+/);
|
||||
if (!complete) paragraphs.pop();
|
||||
const result: string[] = [];
|
||||
let words = 0;
|
||||
for (const paragraph of paragraphs) {
|
||||
const part = paragraph.trim();
|
||||
if (!part) continue;
|
||||
result.push(part);
|
||||
words += part.split(/\s+/u).length;
|
||||
if (words >= 1000) break;
|
||||
}
|
||||
// An unusually large single paragraph must still fit the document contract.
|
||||
// Prefer its last word boundary; ordinary excerpts always keep paragraphs whole.
|
||||
if (result.join("\n\n").length > 56000 || (body.length >= 56000 && words < 1000)) {
|
||||
const prefix = body.slice(0, 56000),
|
||||
at = prefix.lastIndexOf(" ");
|
||||
const bounded = (at > 0 ? prefix.slice(0, at) : prefix).replace(/[\uD800-\uDBFF]$/, "").trim();
|
||||
return { text: bounded, words: bounded ? bounded.split(/\s+/u).length : 0 };
|
||||
}
|
||||
if (words < 1000 && !complete) return null;
|
||||
return { text: result.join("\n\n"), words };
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import { openingExcerpt, type Excerpt } from "./catalog.js";
|
||||
/** User-requested downloads only, from Gutenberg's listed generated-files mirror.
|
||||
* One fetch at a time, at least two seconds apart; coalesce requests and bound the cache. */
|
||||
export function createBookLoader(fetchText: (ebook: number) => Promise<string>, delayMs = 2000) {
|
||||
const cache = new Map<number, { excerpt: Excerpt; bytes: number }>();
|
||||
const pending = new Map<number, Promise<Excerpt>>();
|
||||
let queue = Promise.resolve(),
|
||||
next = 0,
|
||||
bytes = 0;
|
||||
return (ebook: number): Promise<Excerpt> => {
|
||||
const cached = cache.get(ebook);
|
||||
if (cached) {
|
||||
cache.delete(ebook);
|
||||
cache.set(ebook, cached);
|
||||
return Promise.resolve(cached.excerpt);
|
||||
}
|
||||
const existing = pending.get(ebook);
|
||||
if (existing) return existing;
|
||||
if (pending.size >= 8) return Promise.reject(Error("Too many book downloads are queued. Try again shortly."));
|
||||
const task = queue.then(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.max(0, next - Date.now())));
|
||||
next = Date.now() + delayMs;
|
||||
const text = await fetchText(ebook),
|
||||
excerpt = openingExcerpt(text, true)!,
|
||||
size = excerpt.text.length * 2;
|
||||
while (cache.size && (cache.size >= 8 || bytes + size > 32 * 1024 * 1024)) {
|
||||
const key = cache.keys().next().value!;
|
||||
bytes -= cache.get(key)!.bytes;
|
||||
cache.delete(key);
|
||||
}
|
||||
if (size <= 32 * 1024 * 1024) {
|
||||
cache.set(ebook, { excerpt, bytes: size });
|
||||
bytes += size;
|
||||
}
|
||||
return excerpt;
|
||||
});
|
||||
queue = task.then(
|
||||
() => {},
|
||||
() => {},
|
||||
);
|
||||
pending.set(ebook, task);
|
||||
void task.finally(() => pending.delete(ebook)).catch(() => {});
|
||||
return task;
|
||||
};
|
||||
}
|
||||
export const bookURL = (ebook: number) => `https://gutenberg.pglaf.org/cache/epub/${ebook}/pg${ebook}.txt`;
|
||||
export async function fetchBook(ebook: number): Promise<string> {
|
||||
const response = await fetch(bookURL(ebook), {
|
||||
redirect: "error",
|
||||
signal: AbortSignal.timeout(20000),
|
||||
headers: { "User-Agent": "Quixos-Gutenberg-reader/1.0 (on-demand plain-text reader)" },
|
||||
});
|
||||
if (!response.ok)
|
||||
throw Error(
|
||||
response.status === 404
|
||||
? "No UTF-8 plain-text edition is available on this mirror."
|
||||
: `Book download failed (${response.status}). Try again later.`,
|
||||
);
|
||||
if (!response.body) throw Error("Empty book response");
|
||||
const reader = response.body.getReader(),
|
||||
decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
let size = 0,
|
||||
text = "";
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
text += decoder.decode();
|
||||
return text;
|
||||
}
|
||||
size += value.byteLength;
|
||||
if (size > 12 * 1024 * 1024) throw Error("This book exceeds the 12 MiB download limit.");
|
||||
text += decoder.decode(value, { stream: true });
|
||||
if (/^\s*<!doctype html|^\s*<html/i.test(text))
|
||||
throw Error("The mirror returned an HTML page instead of book text.");
|
||||
if (openingExcerpt(text)) return text;
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel();
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import type { PortableImplementation } from "./gen/qx.js";
|
||||
export const handlers: PortableImplementation = {
|
||||
create: ({ ports }) => ports.catalog.create({ name: "Project Gutenberg" }, { key: "catalog" }),
|
||||
};
|
||||
import { createRegistry } from "./gen/qx.js";
|
||||
export const registry = createRegistry(handlers);
|
||||
@@ -0,0 +1,21 @@
|
||||
import { createRegistry } from "./gen/remote.js";
|
||||
import books from "./index-data.js";
|
||||
import { searchIndex } from "./catalog.js";
|
||||
import { createBookLoader, fetchBook } from "./download.js";
|
||||
const search = searchIndex(books),
|
||||
byId = new Map(books.map((book) => [book.ebook, book]));
|
||||
const load = createBookLoader(fetchBook);
|
||||
export const registry = createRegistry({
|
||||
search: ({ input }) => search(input),
|
||||
readBook: async ({ input, signal }) => {
|
||||
const book = byId.get(input.ebook);
|
||||
if (!book) throw Error("Book is not in this catalog snapshot.");
|
||||
if (signal.aborted) throw Error("Book request cancelled.");
|
||||
const excerpt = await load(input.ebook);
|
||||
return {
|
||||
book,
|
||||
url: `https://www.gutenberg.org/ebooks/${book.ebook}`,
|
||||
...excerpt,
|
||||
};
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user