feat(scripts): cached plugin bundle, single-plugin fetch, free-chapter prefix, nightly appender

- load-plugins: bundle once per source revision with an id index; loadPlugin(id)
  loads one plugin (144 ms / 87 MB instead of ~15 s / 1.2 GB).
- fetch-novel: keep only the contiguous run of free chapters (no locked teasers).
- update-novels: append newly free chapters to our EPUBs without moving positions.
- find-novel --json for programs; shared EPUB pieces in epub.js.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
masi
2026-09-26 14:05:51 -03:00
parent 750c2824ce
commit 5c444fc35a
5 changed files with 258 additions and 77 deletions
+13 -38
View File
@@ -2,11 +2,10 @@
// Usage: node scripts/fetch-novel.js <plugin-id> <novel-path> <out-dir>
// Prints the written file path as the last stdout line.
import JSZip from 'jszip';
import { load } from 'cheerio';
import fs from 'fs';
import path from 'path';
import { loadPlugins, print, ROOT } from './load-plugins.js';
import { stripPromo } from './clean-chapter.js';
import { loadPlugin, print } from './load-plugins.js';
import { chapterPage, esc, page, toXhtml } from './epub.js';
// Be gentle with the source site: few parallel requests, retries with backoff.
const CONCURRENCY = 3;
@@ -20,12 +19,7 @@ if (!pluginId || !novelPath || !outDir) {
process.exit(2);
}
const langs = fs
.readdirSync(path.join(ROOT, 'plugins'), { withFileTypes: true })
.filter(d => d.isDirectory() && d.name !== 'multisrc')
.map(d => d.name);
const plugin = (await loadPlugins(langs)).find(p => p.id === pluginId);
if (!plugin) throw new Error(`No plugin with id "${pluginId}"`);
const plugin = await loadPlugin(pluginId);
const retry = async (label, fn) => {
for (let attempt = 1; ; attempt++) {
@@ -39,27 +33,18 @@ const retry = async (label, fn) => {
};
const novel = await retry('novel', () => plugin.parseNovel(novelPath));
const chapters = novel.chapters || [];
if (!chapters.length) throw new Error('Novel has no chapters');
// Plugins mark paid chapters with 🔒 and return only a preview for them. Keep the
// contiguous run of free chapters from the start: a teaser never becomes a chapter, and
// chapters freed later are appended in order by update-novels.js.
const all = novel.chapters || [];
const firstLocked = all.findIndex(c => c.name.includes('🔒'));
const chapters = firstLocked < 0 ? all : all.slice(0, firstLocked);
if (!chapters.length) throw new Error('Novel has no free chapters');
process.stderr.write(
`${novel.name}: ${chapters.length} chapters from ${plugin.name}\n`,
`${novel.name}: ${chapters.length} of ${all.length} chapters from ${plugin.name}` +
(firstLocked < 0 ? '\n' : ' (the rest are locked)\n'),
);
const esc = s =>
String(s ?? '').replace(
/[&<>"]/g,
c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' })[c],
);
// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without
// scripts, remote resources (an EPUB must be self-contained) or promo lines.
const toXhtml = html => {
const $ = load(html, null, false);
$('script, style, iframe, link, meta, img, video, audio, source').remove();
stripPromo($);
return $.xml();
};
const bodies = new Array(chapters.length);
let done = 0;
const queue = chapters.map((c, i) => i);
@@ -87,19 +72,9 @@ zip.file(
</container>`,
);
const page = (title, body) => `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" lang="en" xml:lang="en">
<head><meta charset="UTF-8"/><title>${esc(title)}</title></head>
<body>${body}</body>
</html>`;
const ids = chapters.map((_, i) => `ch${String(i + 1).padStart(4, '0')}`);
chapters.forEach((c, i) => {
zip.file(
`OEBPS/${ids[i]}.xhtml`,
page(c.name, `<h2>${esc(c.name)}</h2>\n${bodies[i]}`),
);
zip.file(`OEBPS/${ids[i]}.xhtml`, chapterPage(c.name, bodies[i]));
});
const toc = chapters