feat(scripts): cached plugin bundle, single-plugin fetch, free-chapter prefix, nightly appender
- load-plugins: bundle once per source revision with an id index; loadPlugin(id) loads one plugin (144 ms / 87 MB instead of ~15 s / 1.2 GB). - fetch-novel: keep only the contiguous run of free chapters (no locked teasers). - update-novels: append newly free chapters to our EPUBs without moving positions. - find-novel --json for programs; shared EPUB pieces in epub.js. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+13
-38
@@ -2,11 +2,10 @@
|
||||
// Usage: node scripts/fetch-novel.js <plugin-id> <novel-path> <out-dir>
|
||||
// Prints the written file path as the last stdout line.
|
||||
import JSZip from 'jszip';
|
||||
import { load } from 'cheerio';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { loadPlugins, print, ROOT } from './load-plugins.js';
|
||||
import { stripPromo } from './clean-chapter.js';
|
||||
import { loadPlugin, print } from './load-plugins.js';
|
||||
import { chapterPage, esc, page, toXhtml } from './epub.js';
|
||||
|
||||
// Be gentle with the source site: few parallel requests, retries with backoff.
|
||||
const CONCURRENCY = 3;
|
||||
@@ -20,12 +19,7 @@ if (!pluginId || !novelPath || !outDir) {
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const langs = fs
|
||||
.readdirSync(path.join(ROOT, 'plugins'), { withFileTypes: true })
|
||||
.filter(d => d.isDirectory() && d.name !== 'multisrc')
|
||||
.map(d => d.name);
|
||||
const plugin = (await loadPlugins(langs)).find(p => p.id === pluginId);
|
||||
if (!plugin) throw new Error(`No plugin with id "${pluginId}"`);
|
||||
const plugin = await loadPlugin(pluginId);
|
||||
|
||||
const retry = async (label, fn) => {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
@@ -39,27 +33,18 @@ const retry = async (label, fn) => {
|
||||
};
|
||||
|
||||
const novel = await retry('novel', () => plugin.parseNovel(novelPath));
|
||||
const chapters = novel.chapters || [];
|
||||
if (!chapters.length) throw new Error('Novel has no chapters');
|
||||
// Plugins mark paid chapters with 🔒 and return only a preview for them. Keep the
|
||||
// contiguous run of free chapters from the start: a teaser never becomes a chapter, and
|
||||
// chapters freed later are appended in order by update-novels.js.
|
||||
const all = novel.chapters || [];
|
||||
const firstLocked = all.findIndex(c => c.name.includes('🔒'));
|
||||
const chapters = firstLocked < 0 ? all : all.slice(0, firstLocked);
|
||||
if (!chapters.length) throw new Error('Novel has no free chapters');
|
||||
process.stderr.write(
|
||||
`${novel.name}: ${chapters.length} chapters from ${plugin.name}\n`,
|
||||
`${novel.name}: ${chapters.length} of ${all.length} chapters from ${plugin.name}` +
|
||||
(firstLocked < 0 ? '\n' : ' (the rest are locked)\n'),
|
||||
);
|
||||
|
||||
const esc = s =>
|
||||
String(s ?? '').replace(
|
||||
/[&<>"]/g,
|
||||
c => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c],
|
||||
);
|
||||
|
||||
// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without
|
||||
// scripts, remote resources (an EPUB must be self-contained) or promo lines.
|
||||
const toXhtml = html => {
|
||||
const $ = load(html, null, false);
|
||||
$('script, style, iframe, link, meta, img, video, audio, source').remove();
|
||||
stripPromo($);
|
||||
return $.xml();
|
||||
};
|
||||
|
||||
const bodies = new Array(chapters.length);
|
||||
let done = 0;
|
||||
const queue = chapters.map((c, i) => i);
|
||||
@@ -87,19 +72,9 @@ zip.file(
|
||||
</container>`,
|
||||
);
|
||||
|
||||
const page = (title, body) => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" lang="en" xml:lang="en">
|
||||
<head><meta charset="UTF-8"/><title>${esc(title)}</title></head>
|
||||
<body>${body}</body>
|
||||
</html>`;
|
||||
|
||||
const ids = chapters.map((_, i) => `ch${String(i + 1).padStart(4, '0')}`);
|
||||
chapters.forEach((c, i) => {
|
||||
zip.file(
|
||||
`OEBPS/${ids[i]}.xhtml`,
|
||||
page(c.name, `<h2>${esc(c.name)}</h2>\n${bodies[i]}`),
|
||||
);
|
||||
zip.file(`OEBPS/${ids[i]}.xhtml`, chapterPage(c.name, bodies[i]));
|
||||
});
|
||||
|
||||
const toc = chapters
|
||||
|
||||
Reference in New Issue
Block a user