diff --git a/scripts/epub.js b/scripts/epub.js
new file mode 100644
index 0000000..18becca
--- /dev/null
+++ b/scripts/epub.js
@@ -0,0 +1,28 @@
+// EPUB pieces shared by fetch-novel.js (new books) and update-novels.js (appends).
+import { load } from 'cheerio';
+import { stripPromo } from './clean-chapter.js';
+
+export const esc = s =>
+ String(s ?? '').replace(
+ /[&<>"]/g,
+ c => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c],
+ );
+
+// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without
+// scripts, remote resources (an EPUB must be self-contained) or promo lines.
+export const toXhtml = html => {
+ const $ = load(html, null, false);
+ $('script, style, iframe, link, meta, img, video, audio, source').remove();
+ stripPromo($);
+ return $.xml();
+};
+
+export const page = (title, body) => `
+
+
+
${esc(title)}
+${body}
+`;
+
+export const chapterPage = (title, body) =>
+ page(title, `${esc(title)}
\n${body}`);
diff --git a/scripts/fetch-novel.js b/scripts/fetch-novel.js
index 7921b82..56da531 100644
--- a/scripts/fetch-novel.js
+++ b/scripts/fetch-novel.js
@@ -2,11 +2,10 @@
// Usage: node scripts/fetch-novel.js
// Prints the written file path as the last stdout line.
import JSZip from 'jszip';
-import { load } from 'cheerio';
import fs from 'fs';
import path from 'path';
-import { loadPlugins, print, ROOT } from './load-plugins.js';
-import { stripPromo } from './clean-chapter.js';
+import { loadPlugin, print } from './load-plugins.js';
+import { chapterPage, esc, page, toXhtml } from './epub.js';
// Be gentle with the source site: few parallel requests, retries with backoff.
const CONCURRENCY = 3;
@@ -20,12 +19,7 @@ if (!pluginId || !novelPath || !outDir) {
process.exit(2);
}
-const langs = fs
- .readdirSync(path.join(ROOT, 'plugins'), { withFileTypes: true })
- .filter(d => d.isDirectory() && d.name !== 'multisrc')
- .map(d => d.name);
-const plugin = (await loadPlugins(langs)).find(p => p.id === pluginId);
-if (!plugin) throw new Error(`No plugin with id "${pluginId}"`);
+const plugin = await loadPlugin(pluginId);
const retry = async (label, fn) => {
for (let attempt = 1; ; attempt++) {
@@ -39,27 +33,18 @@ const retry = async (label, fn) => {
};
const novel = await retry('novel', () => plugin.parseNovel(novelPath));
-const chapters = novel.chapters || [];
-if (!chapters.length) throw new Error('Novel has no chapters');
+// Plugins mark paid chapters with 🔒 and return only a preview for them. Keep the
+// contiguous run of free chapters from the start: a teaser never becomes a chapter, and
+// chapters freed later are appended in order by update-novels.js.
+const all = novel.chapters || [];
+const firstLocked = all.findIndex(c => c.name.includes('🔒'));
+const chapters = firstLocked < 0 ? all : all.slice(0, firstLocked);
+if (!chapters.length) throw new Error('Novel has no free chapters');
process.stderr.write(
- `${novel.name}: ${chapters.length} chapters from ${plugin.name}\n`,
+ `${novel.name}: ${chapters.length} of ${all.length} chapters from ${plugin.name}` +
+ (firstLocked < 0 ? '\n' : ' (the rest are locked)\n'),
);
-const esc = s =>
- String(s ?? '').replace(
- /[&<>"]/g,
- c => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c],
- );
-
-// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without
-// scripts, remote resources (an EPUB must be self-contained) or promo lines.
-const toXhtml = html => {
- const $ = load(html, null, false);
- $('script, style, iframe, link, meta, img, video, audio, source').remove();
- stripPromo($);
- return $.xml();
-};
-
const bodies = new Array(chapters.length);
let done = 0;
const queue = chapters.map((c, i) => i);
@@ -87,19 +72,9 @@ zip.file(
`,
);
-const page = (title, body) => `
-
-
-${esc(title)}
-${body}
-`;
-
const ids = chapters.map((_, i) => `ch${String(i + 1).padStart(4, '0')}`);
chapters.forEach((c, i) => {
- zip.file(
- `OEBPS/${ids[i]}.xhtml`,
- page(c.name, `${esc(c.name)}
\n${bodies[i]}`),
- );
+ zip.file(`OEBPS/${ids[i]}.xhtml`, chapterPage(c.name, bodies[i]));
});
const toc = chapters
diff --git a/scripts/find-novel.js b/scripts/find-novel.js
index 3b4f01a..73eee42 100644
--- a/scripts/find-novel.js
+++ b/scripts/find-novel.js
@@ -1,13 +1,16 @@
// Search every plugin of the given languages for a title and list the sources
-// that carry it. Usage: node scripts/find-novel.js "" [lang...]
-// Languages are plugin folder names (default: english multi).
+// that carry it. Usage: node scripts/find-novel.js [--json] "" [lang...]
+// Languages are plugin folder names (default: english multi). --json prints one JSON
+// array of {source, id, name, path, url, exact} for programs (the request page).
import { loadPlugins, print } from './load-plugins.js';
const TIMEOUT_MS = 20_000;
const CONCURRENCY = 16;
const PER_SOURCE = 3;
-const [term, ...langArgs] = process.argv.slice(2);
+const args = process.argv.slice(2);
+const asJson = args[0] === '--json';
+const [term, ...langArgs] = asJson ? args.slice(1) : args;
if (!term) {
console.error('Usage: node scripts/find-novel.js "" [lang...]');
process.exit(2);
@@ -64,6 +67,10 @@ const matches = hits.filter(h =>
const rank = h => (norm(h.name) === exact ? 0 : 1);
matches.sort((a, b) => rank(a) - rank(b) || a.source.localeCompare(b.source));
+if (asJson) {
+ print(JSON.stringify(matches.map(h => ({ ...h, exact: rank(h) === 0 }))));
+ process.exit(0);
+}
for (const h of matches) {
print(
`${rank(h) === 0 ? '*' : ' '} ${h.source.padEnd(28)} ${h.name}\n` +
diff --git a/scripts/load-plugins.js b/scripts/load-plugins.js
index d93d090..1267474 100644
--- a/scripts/load-plugins.js
+++ b/scripts/load-plugins.js
@@ -1,6 +1,10 @@
// Load compiled plugin instances outside the app, for the novel CLI scripts.
+// Bundling ~165 plugins takes ~15 s and each bundle carries its own libraries, so the
+// bundle is built once per source revision and cached, with an id -> file index:
+// fetching a novel then loads a single plugin instead of all of them.
import * as esbuild from 'esbuild';
import { execFileSync } from 'child_process';
+import { createHash } from 'crypto';
import { createRequire } from 'module';
import { fileURLToPath } from 'url';
import path, { dirname } from 'path';
@@ -9,60 +13,114 @@ import os from 'os';
const require = createRequire(import.meta.url);
export const ROOT = path.resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const CACHE =
+ process.env.NOVEL_PLUGIN_CACHE ||
+ path.join(os.homedir(), '.cache', 'novel-plugins');
+export const LANGS = ['english', 'multi'];
// The shared fetch helper logs every request with its headers (credentials
// included). Callers print their output explicitly with `print`.
export const print = (...args) => process.stdout.write(args.join(' ') + '\n');
console.log = () => undefined;
-/** Bundle and instantiate every plugin in the given language folders. */
-export async function loadPlugins(langs) {
+const git = args =>
+ execFileSync('git', args, { cwd: ROOT, encoding: 'utf8' }).trim();
+
+/** Cache key: the commit plus any uncommitted changes to plugins or libs. */
+function revision(langs) {
+ let head = 'nogit';
+ let dirty = '';
+ try {
+ head = git(['rev-parse', 'HEAD']);
+ dirty = git(['diff', 'HEAD', '--', 'plugins', 'src']);
+ } catch {
+ // Not a git checkout: the cache is keyed by language set only.
+ }
+ return createHash('sha256')
+ .update(`${head}\n${dirty}\n${langs.join(',')}`)
+ .digest('hex')
+ .slice(0, 16);
+}
+
+/** Build (once per revision) and return the cache dir holding bundles + index.json. */
+async function bundle(langs) {
+ const dir = path.join(CACHE, revision(langs));
+ if (fs.existsSync(path.join(dir, 'index.json'))) return dir;
+
// Multisrc plugins (Madara, NovelFire, ...) are generated, not committed.
execFileSync('node', ['plugins/multisrc/generate-multisrc-plugins.js'], {
cwd: ROOT,
stdio: 'ignore',
});
-
const entryPoints = langs.flatMap(lang => {
- const dir = path.join(ROOT, 'plugins', lang);
- if (!fs.existsSync(dir))
+ const src = path.join(ROOT, 'plugins', lang);
+ if (!fs.existsSync(src))
throw new Error(`No plugin folder: plugins/${lang}`);
return fs
- .readdirSync(dir)
+ .readdirSync(src)
.filter(f => f.endsWith('.ts'))
- .map(f => path.join(dir, f));
+ .map(f => path.join(src, f));
});
- const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-plugins-'));
- try {
- const result = await esbuild.build({
- entryPoints,
- outdir,
- bundle: true,
- platform: 'node',
- format: 'cjs',
- target: 'node22',
- outExtension: { '.js': '.cjs' },
- logLevel: 'silent',
- alias: {
- '@libs': path.join(ROOT, 'src/libs'),
- '@': path.join(ROOT, 'src'),
- },
- });
- if (result.errors.length) throw new Error('Plugin bundling failed');
+ const tmp = `${dir}.tmp-${process.pid}`;
+ fs.rmSync(tmp, { recursive: true, force: true });
+ const result = await esbuild.build({
+ entryPoints,
+ outdir: tmp,
+ bundle: true,
+ platform: 'node',
+ format: 'cjs',
+ target: 'node22',
+ outExtension: { '.js': '.cjs' },
+ logLevel: 'silent',
+ alias: {
+ '@libs': path.join(ROOT, 'src/libs'),
+ '@': path.join(ROOT, 'src'),
+ },
+ });
+ if (result.errors.length) throw new Error('Plugin bundling failed');
- const plugins = [];
- for (const file of fs.readdirSync(outdir, { recursive: true })) {
- if (!file.endsWith('.cjs')) continue;
- try {
- const mod = require(path.join(outdir, file));
- plugins.push(mod.default ?? mod);
- } catch {
- // A plugin that throws on load is skipped, like a broken source in the app.
- }
+ const index = {};
+ for (const file of fs.readdirSync(tmp, { recursive: true })) {
+ if (!file.endsWith('.cjs')) continue;
+ try {
+ const mod = require(path.join(tmp, file));
+ const plugin = mod.default ?? mod;
+ if (plugin?.id) index[plugin.id] = file;
+ } catch {
+ // A plugin that throws on load is skipped, like a broken source in the app.
}
- return plugins;
- } finally {
- fs.rmSync(outdir, { recursive: true, force: true });
}
+ fs.writeFileSync(path.join(tmp, 'index.json'), JSON.stringify(index));
+ fs.rmSync(dir, { recursive: true, force: true });
+ fs.renameSync(tmp, dir);
+ return dir;
+}
+
+function instantiate(dir, file) {
+ const mod = require(path.join(dir, file));
+ return mod.default ?? mod;
+}
+
+/** Every plugin in the given language folders (for searching across sources). */
+export async function loadPlugins(langs = LANGS) {
+ const dir = await bundle(langs);
+ const index = JSON.parse(fs.readFileSync(path.join(dir, 'index.json')));
+ const plugins = [];
+ for (const file of Object.values(index)) {
+ try {
+ plugins.push(instantiate(dir, file));
+ } catch {
+ // Skipped, as at bundle time.
+ }
+ }
+ return plugins;
+}
+
+/** One plugin by id, loading nothing else. */
+export async function loadPlugin(id, langs = LANGS) {
+ const dir = await bundle(langs);
+ const file = JSON.parse(fs.readFileSync(path.join(dir, 'index.json')))[id];
+ if (!file) throw new Error(`No plugin with id "${id}"`);
+ return instantiate(dir, file);
}
diff --git a/scripts/update-novels.js b/scripts/update-novels.js
new file mode 100644
index 0000000..d3e85df
--- /dev/null
+++ b/scripts/update-novels.js
@@ -0,0 +1,113 @@
+// Append newly published (and newly free) chapters to EPUBs built by fetch-novel.js.
+// Usage: node scripts/update-novels.js
+// Only EPUBs whose identifier is urn:lnreader:: are touched. Chapters are
+// appended at the end, so existing positions (Kavita pages, reading progress,
+// illustration anchors) never move. Prints one line per updated novel.
+import JSZip from 'jszip';
+import { load } from 'cheerio';
+import fs from 'fs';
+import path from 'path';
+import { loadPlugin, print } from './load-plugins.js';
+import { chapterPage, esc, toXhtml } from './epub.js';
+
+const CONCURRENCY = 3;
+const RETRIES = 4;
+
+const [library] = process.argv.slice(2);
+if (!library) {
+ console.error('Usage: node scripts/update-novels.js ');
+ process.exit(2);
+}
+
+const retry = async (label, fn) => {
+ for (let attempt = 1; ; attempt++) {
+ try {
+ return await fn();
+ } catch (e) {
+ if (attempt >= RETRIES) throw new Error(`${label}: ${e.message}`);
+ await new Promise(r => setTimeout(r, 1000 * 2 ** attempt));
+ }
+ }
+};
+
+async function update(file) {
+ const zip = await JSZip.loadAsync(fs.readFileSync(file));
+ const opfPath = 'OEBPS/content.opf';
+ if (!zip.file(opfPath)) return;
+ const opf = load(await zip.file(opfPath).async('string'), { xml: true });
+ const uid = opf('dc\\:identifier').first().text();
+ const m = uid.match(/^urn:lnreader:([^:]+):(.+)$/);
+ if (!m) return;
+ const [, pluginId, novelPath] = m;
+ const have = opf('spine itemref').length;
+
+ const plugin = await loadPlugin(pluginId);
+ const novel = await retry('novel', () => plugin.parseNovel(novelPath));
+ const all = novel.chapters || [];
+ const firstLocked = all.findIndex(c => c.name.includes('🔒'));
+ const free = firstLocked < 0 ? all : all.slice(0, firstLocked);
+ if (free.length <= have) return;
+
+ const added = free.slice(have);
+ const bodies = new Array(added.length);
+ const queue = added.map((_, i) => i);
+ await Promise.all(
+ Array.from({ length: CONCURRENCY }, async () => {
+ for (let i; (i = queue.shift()) !== undefined;) {
+ bodies[i] = toXhtml(
+ await retry(added[i].name, () => plugin.parseChapter(added[i].path)),
+ );
+ }
+ }),
+ );
+
+ const nav = load(await zip.file('OEBPS/nav.xhtml').async('string'), {
+ xml: true,
+ });
+ added.forEach((c, i) => {
+ const id = `ch${String(have + i + 1).padStart(4, '0')}`;
+ zip.file(`OEBPS/${id}.xhtml`, chapterPage(c.name, bodies[i]));
+ opf('manifest').append(
+ ` `,
+ );
+ opf('spine').append(``);
+ nav('nav ol')
+ .first()
+ .append(`${esc(c.name)}`);
+ });
+ opf('meta[property="dcterms:modified"]').text(
+ new Date().toISOString().replace(/\.\d+Z$/, 'Z'),
+ );
+ zip.file(opfPath, opf.xml());
+ zip.file('OEBPS/nav.xhtml', nav.xml());
+ // EPUB requires mimetype stored uncompressed; loaded entries lose that option.
+ zip.file('mimetype', 'application/epub+zip', { compression: 'STORE' });
+ const tmp = `${file}.tmp`;
+ fs.writeFileSync(
+ tmp,
+ await zip.generateAsync({
+ type: 'nodebuffer',
+ compression: 'DEFLATE',
+ mimeType: 'application/epub+zip',
+ }),
+ );
+ fs.renameSync(tmp, file);
+ print(
+ `${path.basename(file)}: +${added.length} chapters (${have} -> ${free.length})`,
+ );
+}
+
+const epubs = fs
+ .readdirSync(library, { recursive: true })
+ .filter(f => f.endsWith('.epub'))
+ .map(f => path.join(library, f));
+let failed = 0;
+for (const file of epubs) {
+ try {
+ await update(file);
+ } catch (e) {
+ failed++;
+ process.stderr.write(`${path.basename(file)}: ${e.message}\n`);
+ }
+}
+process.exit(failed ? 1 : 0);