feat(scripts): cached plugin bundle, single-plugin fetch, free-chapter prefix, nightly appender

- load-plugins: bundle once per source revision with an id index; loadPlugin(id)
  loads one plugin (144 ms / 87 MB instead of ~15 s / 1.2 GB).
- fetch-novel: keep only the contiguous run of free chapters (no locked teasers).
- update-novels: append newly free chapters to our EPUBs without moving positions.
- find-novel --json for programs; shared EPUB pieces in epub.js.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/epub.js

@@ -0,0 +1,28 @@
+// EPUB pieces shared by fetch-novel.js (new books) and update-novels.js (appends).
+import { load } from 'cheerio';
+import { stripPromo } from './clean-chapter.js';
+
+export const esc = s =>
+  String(s ?? '').replace(
+    /[&<>"]/g,
+    c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' })[c],
+  );
+
+// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without
+// scripts, remote resources (an EPUB must be self-contained) or promo lines.
+export const toXhtml = html => {
+  const $ = load(html, null, false);
+  $('script, style, iframe, link, meta, img, video, audio, source').remove();
+  stripPromo($);
+  return $.xml();
+};
+
+export const page = (title, body) => `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" lang="en" xml:lang="en">
+<head><meta charset="UTF-8"/><title>${esc(title)}</title></head>
+<body>${body}</body>
+</html>`;
+
+export const chapterPage = (title, body) =>
+  page(title, `<h2>${esc(title)}</h2>\n${body}`);

scripts/fetch-novel.js

@@ -2,11 +2,10 @@
 // Usage: node scripts/fetch-novel.js <plugin-id> <novel-path> <out-dir>
 // Prints the written file path as the last stdout line.
 import JSZip from 'jszip';
-import { load } from 'cheerio';
 import fs from 'fs';
 import path from 'path';
-import { loadPlugins, print, ROOT } from './load-plugins.js';
+import { loadPlugin, print } from './load-plugins.js';
-import { stripPromo } from './clean-chapter.js';
+import { chapterPage, esc, page, toXhtml } from './epub.js';
 
 // Be gentle with the source site: few parallel requests, retries with backoff.
 const CONCURRENCY = 3;
@@ -20,12 +19,7 @@ if (!pluginId || !novelPath || !outDir) {
   process.exit(2);
 }
 
-const langs = fs
+const plugin = await loadPlugin(pluginId);
-  .readdirSync(path.join(ROOT, 'plugins'), { withFileTypes: true })
-  .filter(d => d.isDirectory() && d.name !== 'multisrc')
-  .map(d => d.name);
-const plugin = (await loadPlugins(langs)).find(p => p.id === pluginId);
-if (!plugin) throw new Error(`No plugin with id "${pluginId}"`);
 
 const retry = async (label, fn) => {
   for (let attempt = 1; ; attempt++) {
@@ -39,27 +33,18 @@ const retry = async (label, fn) => {
 };
 
 const novel = await retry('novel', () => plugin.parseNovel(novelPath));
-const chapters = novel.chapters || [];
+// Plugins mark paid chapters with 🔒 and return only a preview for them. Keep the
-if (!chapters.length) throw new Error('Novel has no chapters');
+// contiguous run of free chapters from the start: a teaser never becomes a chapter, and
+// chapters freed later are appended in order by update-novels.js.
+const all = novel.chapters || [];
+const firstLocked = all.findIndex(c => c.name.includes('🔒'));
+const chapters = firstLocked < 0 ? all : all.slice(0, firstLocked);
+if (!chapters.length) throw new Error('Novel has no free chapters');
 process.stderr.write(
-  `${novel.name}: ${chapters.length} chapters from ${plugin.name}\n`,
+  `${novel.name}: ${chapters.length} of ${all.length} chapters from ${plugin.name}` +
+    (firstLocked < 0 ? '\n' : ' (the rest are locked)\n'),
 );
 
-const esc = s =>
-  String(s ?? '').replace(
-    /[&<>"]/g,
-    c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' })[c],
-  );
-
-// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without
-// scripts, remote resources (an EPUB must be self-contained) or promo lines.
-const toXhtml = html => {
-  const $ = load(html, null, false);
-  $('script, style, iframe, link, meta, img, video, audio, source').remove();
-  stripPromo($);
-  return $.xml();
-};
-
 const bodies = new Array(chapters.length);
 let done = 0;
 const queue = chapters.map((c, i) => i);
@@ -87,19 +72,9 @@ zip.file(
 </container>`,
 );
 
-const page = (title, body) => `<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" lang="en" xml:lang="en">
-<head><meta charset="UTF-8"/><title>${esc(title)}</title></head>
-<body>${body}</body>
-</html>`;
-
 const ids = chapters.map((_, i) => `ch${String(i + 1).padStart(4, '0')}`);
 chapters.forEach((c, i) => {
-  zip.file(
+  zip.file(`OEBPS/${ids[i]}.xhtml`, chapterPage(c.name, bodies[i]));
-    `OEBPS/${ids[i]}.xhtml`,
-    page(c.name, `<h2>${esc(c.name)}</h2>\n${bodies[i]}`),
-  );
 });
 
 const toc = chapters

scripts/find-novel.js

@@ -1,13 +1,16 @@
 // Search every plugin of the given languages for a title and list the sources
-// that carry it. Usage: node scripts/find-novel.js "<title>" [lang...]
+// that carry it. Usage: node scripts/find-novel.js [--json] "<title>" [lang...]
-// Languages are plugin folder names (default: english multi).
+// Languages are plugin folder names (default: english multi). --json prints one JSON
+// array of {source, id, name, path, url, exact} for programs (the request page).
 import { loadPlugins, print } from './load-plugins.js';
 
 const TIMEOUT_MS = 20_000;
 const CONCURRENCY = 16;
 const PER_SOURCE = 3;
 
-const [term, ...langArgs] = process.argv.slice(2);
+const args = process.argv.slice(2);
+const asJson = args[0] === '--json';
+const [term, ...langArgs] = asJson ? args.slice(1) : args;
 if (!term) {
   console.error('Usage: node scripts/find-novel.js "<title>" [lang...]');
   process.exit(2);
@@ -64,6 +67,10 @@ const matches = hits.filter(h =>
 const rank = h => (norm(h.name) === exact ? 0 : 1);
 matches.sort((a, b) => rank(a) - rank(b) || a.source.localeCompare(b.source));
 
+if (asJson) {
+  print(JSON.stringify(matches.map(h => ({ ...h, exact: rank(h) === 0 }))));
+  process.exit(0);
+}
 for (const h of matches) {
   print(
     `${rank(h) === 0 ? '*' : ' '} ${h.source.padEnd(28)} ${h.name}\n` +

scripts/load-plugins.js

@@ -1,6 +1,10 @@
 // Load compiled plugin instances outside the app, for the novel CLI scripts.
+// Bundling ~165 plugins takes ~15 s and each bundle carries its own libraries, so the
+// bundle is built once per source revision and cached, with an id -> file index:
+// fetching a novel then loads a single plugin instead of all of them.
 import * as esbuild from 'esbuild';
 import { execFileSync } from 'child_process';
+import { createHash } from 'crypto';
 import { createRequire } from 'module';
 import { fileURLToPath } from 'url';
 import path, { dirname } from 'path';
@@ -9,60 +13,114 @@ import os from 'os';
 
 const require = createRequire(import.meta.url);
 export const ROOT = path.resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const CACHE =
+  process.env.NOVEL_PLUGIN_CACHE ||
+  path.join(os.homedir(), '.cache', 'novel-plugins');
+export const LANGS = ['english', 'multi'];
 
 // The shared fetch helper logs every request with its headers (credentials
 // included). Callers print their output explicitly with `print`.
 export const print = (...args) => process.stdout.write(args.join(' ') + '\n');
 console.log = () => undefined;
 
-/** Bundle and instantiate every plugin in the given language folders. */
+const git = args =>
-export async function loadPlugins(langs) {
+  execFileSync('git', args, { cwd: ROOT, encoding: 'utf8' }).trim();
+
+/** Cache key: the commit plus any uncommitted changes to plugins or libs. */
+function revision(langs) {
+  let head = 'nogit';
+  let dirty = '';
+  try {
+    head = git(['rev-parse', 'HEAD']);
+    dirty = git(['diff', 'HEAD', '--', 'plugins', 'src']);
+  } catch {
+    // Not a git checkout: the cache is keyed by language set only.
+  }
+  return createHash('sha256')
+    .update(`${head}\n${dirty}\n${langs.join(',')}`)
+    .digest('hex')
+    .slice(0, 16);
+}
+
+/** Build (once per revision) and return the cache dir holding bundles + index.json. */
+async function bundle(langs) {
+  const dir = path.join(CACHE, revision(langs));
+  if (fs.existsSync(path.join(dir, 'index.json'))) return dir;
+
   // Multisrc plugins (Madara, NovelFire, ...) are generated, not committed.
   execFileSync('node', ['plugins/multisrc/generate-multisrc-plugins.js'], {
     cwd: ROOT,
     stdio: 'ignore',
   });
-
   const entryPoints = langs.flatMap(lang => {
-    const dir = path.join(ROOT, 'plugins', lang);
+    const src = path.join(ROOT, 'plugins', lang);
-    if (!fs.existsSync(dir))
+    if (!fs.existsSync(src))
       throw new Error(`No plugin folder: plugins/${lang}`);
     return fs
-      .readdirSync(dir)
+      .readdirSync(src)
       .filter(f => f.endsWith('.ts'))
-      .map(f => path.join(dir, f));
+      .map(f => path.join(src, f));
   });
 
-  const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-plugins-'));
+  const tmp = `${dir}.tmp-${process.pid}`;
-  try {
+  fs.rmSync(tmp, { recursive: true, force: true });
-    const result = await esbuild.build({
+  const result = await esbuild.build({
-      entryPoints,
+    entryPoints,
-      outdir,
+    outdir: tmp,
-      bundle: true,
+    bundle: true,
-      platform: 'node',
+    platform: 'node',
-      format: 'cjs',
+    format: 'cjs',
-      target: 'node22',
+    target: 'node22',
-      outExtension: { '.js': '.cjs' },
+    outExtension: { '.js': '.cjs' },
-      logLevel: 'silent',
+    logLevel: 'silent',
-      alias: {
+    alias: {
-        '@libs': path.join(ROOT, 'src/libs'),
+      '@libs': path.join(ROOT, 'src/libs'),
-        '@': path.join(ROOT, 'src'),
+      '@': path.join(ROOT, 'src'),
-      },
+    },
-    });
+  });
-    if (result.errors.length) throw new Error('Plugin bundling failed');
+  if (result.errors.length) throw new Error('Plugin bundling failed');
 
-    const plugins = [];
+  const index = {};
-    for (const file of fs.readdirSync(outdir, { recursive: true })) {
+  for (const file of fs.readdirSync(tmp, { recursive: true })) {
-      if (!file.endsWith('.cjs')) continue;
+    if (!file.endsWith('.cjs')) continue;
-      try {
+    try {
-        const mod = require(path.join(outdir, file));
+      const mod = require(path.join(tmp, file));
-        plugins.push(mod.default ?? mod);
+      const plugin = mod.default ?? mod;
-      } catch {
+      if (plugin?.id) index[plugin.id] = file;
-        // A plugin that throws on load is skipped, like a broken source in the app.
+    } catch {
-      }
+      // A plugin that throws on load is skipped, like a broken source in the app.
     }
-    return plugins;
-  } finally {
-    fs.rmSync(outdir, { recursive: true, force: true });
   }
+  fs.writeFileSync(path.join(tmp, 'index.json'), JSON.stringify(index));
+  fs.rmSync(dir, { recursive: true, force: true });
+  fs.renameSync(tmp, dir);
+  return dir;
+}
+
+function instantiate(dir, file) {
+  const mod = require(path.join(dir, file));
+  return mod.default ?? mod;
+}
+
+/** Every plugin in the given language folders (for searching across sources). */
+export async function loadPlugins(langs = LANGS) {
+  const dir = await bundle(langs);
+  const index = JSON.parse(fs.readFileSync(path.join(dir, 'index.json')));
+  const plugins = [];
+  for (const file of Object.values(index)) {
+    try {
+      plugins.push(instantiate(dir, file));
+    } catch {
+      // Skipped, as at bundle time.
+    }
+  }
+  return plugins;
+}
+
+/** One plugin by id, loading nothing else. */
+export async function loadPlugin(id, langs = LANGS) {
+  const dir = await bundle(langs);
+  const file = JSON.parse(fs.readFileSync(path.join(dir, 'index.json')))[id];
+  if (!file) throw new Error(`No plugin with id "${id}"`);
+  return instantiate(dir, file);
 }

scripts/update-novels.js

@@ -0,0 +1,113 @@
+// Append newly published (and newly free) chapters to EPUBs built by fetch-novel.js.
+// Usage: node scripts/update-novels.js <library-dir>
+// Only EPUBs whose identifier is urn:lnreader:<plugin>:<path> are touched. Chapters are
+// appended at the end, so existing positions (Kavita pages, reading progress,
+// illustration anchors) never move. Prints one line per updated novel.
+import JSZip from 'jszip';
+import { load } from 'cheerio';
+import fs from 'fs';
+import path from 'path';
+import { loadPlugin, print } from './load-plugins.js';
+import { chapterPage, esc, toXhtml } from './epub.js';
+
+const CONCURRENCY = 3;
+const RETRIES = 4;
+
+const [library] = process.argv.slice(2);
+if (!library) {
+  console.error('Usage: node scripts/update-novels.js <library-dir>');
+  process.exit(2);
+}
+
+const retry = async (label, fn) => {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await fn();
+    } catch (e) {
+      if (attempt >= RETRIES) throw new Error(`${label}: ${e.message}`);
+      await new Promise(r => setTimeout(r, 1000 * 2 ** attempt));
+    }
+  }
+};
+
+async function update(file) {
+  const zip = await JSZip.loadAsync(fs.readFileSync(file));
+  const opfPath = 'OEBPS/content.opf';
+  if (!zip.file(opfPath)) return;
+  const opf = load(await zip.file(opfPath).async('string'), { xml: true });
+  const uid = opf('dc\\:identifier').first().text();
+  const m = uid.match(/^urn:lnreader:([^:]+):(.+)$/);
+  if (!m) return;
+  const [, pluginId, novelPath] = m;
+  const have = opf('spine itemref').length;
+
+  const plugin = await loadPlugin(pluginId);
+  const novel = await retry('novel', () => plugin.parseNovel(novelPath));
+  const all = novel.chapters || [];
+  const firstLocked = all.findIndex(c => c.name.includes('🔒'));
+  const free = firstLocked < 0 ? all : all.slice(0, firstLocked);
+  if (free.length <= have) return;
+
+  const added = free.slice(have);
+  const bodies = new Array(added.length);
+  const queue = added.map((_, i) => i);
+  await Promise.all(
+    Array.from({ length: CONCURRENCY }, async () => {
+      for (let i; (i = queue.shift()) !== undefined;) {
+        bodies[i] = toXhtml(
+          await retry(added[i].name, () => plugin.parseChapter(added[i].path)),
+        );
+      }
+    }),
+  );
+
+  const nav = load(await zip.file('OEBPS/nav.xhtml').async('string'), {
+    xml: true,
+  });
+  added.forEach((c, i) => {
+    const id = `ch${String(have + i + 1).padStart(4, '0')}`;
+    zip.file(`OEBPS/${id}.xhtml`, chapterPage(c.name, bodies[i]));
+    opf('manifest').append(
+      `<item id="${id}" href="${id}.xhtml" media-type="application/xhtml+xml"/>`,
+    );
+    opf('spine').append(`<itemref idref="${id}"/>`);
+    nav('nav ol')
+      .first()
+      .append(`<li><a href="${id}.xhtml">${esc(c.name)}</a></li>`);
+  });
+  opf('meta[property="dcterms:modified"]').text(
+    new Date().toISOString().replace(/\.\d+Z$/, 'Z'),
+  );
+  zip.file(opfPath, opf.xml());
+  zip.file('OEBPS/nav.xhtml', nav.xml());
+  // EPUB requires mimetype stored uncompressed; loaded entries lose that option.
+  zip.file('mimetype', 'application/epub+zip', { compression: 'STORE' });
+  const tmp = `${file}.tmp`;
+  fs.writeFileSync(
+    tmp,
+    await zip.generateAsync({
+      type: 'nodebuffer',
+      compression: 'DEFLATE',
+      mimeType: 'application/epub+zip',
+    }),
+  );
+  fs.renameSync(tmp, file);
+  print(
+    `${path.basename(file)}: +${added.length} chapters (${have} -> ${free.length})`,
+  );
+}
+
+const epubs = fs
+  .readdirSync(library, { recursive: true })
+  .filter(f => f.endsWith('.epub'))
+  .map(f => path.join(library, f));
+let failed = 0;
+for (const file of epubs) {
+  try {
+    await update(file);
+  } catch (e) {
+    failed++;
+    process.stderr.write(`${path.basename(file)}: ${e.message}\n`);
+  }
+}
+process.exit(failed ? 1 : 0);