From fba9a35075ef84cfa67b840334072ccbd7271586 Mon Sep 17 00:00:00 2001 From: masi Date: Fri, 25 Sep 2026 13:40:23 -0300 Subject: [PATCH] feat: fetch-novel builds an EPUB from any plugin; Kavita TOC fix - scripts/load-plugins.js: shared plugin loader for the novel CLI scripts - scripts/fetch-novel.js: download a novel via its plugin into a clean EPUB 3 - find-novel prints the plugin id/path for each match - kavita 1.0.1: innermost TOC entry names the chapter Co-Authored-By: Claude Opus 5.5 --- plugins/multi/kavita.ts | 6 +- scripts/fetch-novel.js | 178 ++++++++++++++++++++++++++++++++++++++++ scripts/find-novel.js | 161 +++++++++++++----------------------- scripts/load-plugins.js | 68 +++++++++++++++ 4 files changed, 307 insertions(+), 106 deletions(-) create mode 100644 scripts/fetch-novel.js create mode 100644 scripts/load-plugins.js diff --git a/plugins/multi/kavita.ts b/plugins/multi/kavita.ts index e62800e..cb6a0bd 100644 --- a/plugins/multi/kavita.ts +++ b/plugins/multi/kavita.ts @@ -37,7 +37,7 @@ class KavitaPlugin implements Plugin.PluginBase { id = 'kavita'; name = 'Kavita'; icon = 'src/multi/kavita/icon.png'; - version = '1.0.0'; + version = '1.0.1'; site = 'https://www.kavitareader.com'; pluginSettings = { @@ -166,10 +166,12 @@ class KavitaPlugin implements Plugin.PluginBase { for (const book of books) { const toc = await this.json(`Book/${book.id}/chapters`); const titles = new Map(); + // Innermost entry wins: a "Part"/"Contents" parent often shares its + // page with the first chapter below it. const walk = (items: TocItem[]) => items.forEach(item => { - if (!titles.has(item.page)) titles.set(item.page, item.title); if (item.children?.length) walk(item.children); + if (!titles.has(item.page)) titles.set(item.page, item.title); }); walk(toc); diff --git a/scripts/fetch-novel.js b/scripts/fetch-novel.js new file mode 100644 index 0000000..94bf022 --- /dev/null +++ b/scripts/fetch-novel.js @@ -0,0 +1,178 @@ +// Download a whole novel through its plugin and write it as an EPUB 3 file. +// Usage: node scripts/fetch-novel.js +// Prints the written file path as the last stdout line. +import JSZip from 'jszip'; +import { load } from 'cheerio'; +import fs from 'fs'; +import path from 'path'; +import { loadPlugins, print, ROOT } from './load-plugins.js'; + +// Be gentle with the source site: few parallel requests, retries with backoff. +const CONCURRENCY = 3; +const RETRIES = 4; + +const [pluginId, novelPath, outDir] = process.argv.slice(2); +if (!pluginId || !novelPath || !outDir) { + console.error( + 'Usage: node scripts/fetch-novel.js ', + ); + process.exit(2); +} + +const langs = fs + .readdirSync(path.join(ROOT, 'plugins'), { withFileTypes: true }) + .filter(d => d.isDirectory() && d.name !== 'multisrc') + .map(d => d.name); +const plugin = (await loadPlugins(langs)).find(p => p.id === pluginId); +if (!plugin) throw new Error(`No plugin with id "${pluginId}"`); + +const retry = async (label, fn) => { + for (let attempt = 1; ; attempt++) { + try { + return await fn(); + } catch (e) { + if (attempt >= RETRIES) throw new Error(`${label}: ${e.message}`); + await new Promise(r => setTimeout(r, 1000 * 2 ** attempt)); + } + } +}; + +const novel = await retry('novel', () => plugin.parseNovel(novelPath)); +const chapters = novel.chapters || []; +if (!chapters.length) throw new Error('Novel has no chapters'); +process.stderr.write( + `${novel.name}: ${chapters.length} chapters from ${plugin.name}\n`, +); + +const esc = s => + String(s ?? '').replace( + /[&<>"]/g, + c => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c], + ); + +// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without +// scripts or remote resources (an EPUB must be self-contained). +const toXhtml = html => { + const $ = load(html, null, false); + $('script, style, iframe, link, meta, img, video, audio, source').remove(); + return $.xml(); +}; + +const bodies = new Array(chapters.length); +let done = 0; +const queue = chapters.map((c, i) => i); +await Promise.all( + Array.from({ length: CONCURRENCY }, async () => { + for (let i; (i = queue.shift()) !== undefined;) { + const html = await retry(chapters[i].name, () => + plugin.parseChapter(chapters[i].path), + ); + bodies[i] = toXhtml(html); + if (++done % 50 === 0 || done === chapters.length) { + process.stderr.write(` ${done}/${chapters.length}\n`); + } + } + }), +); + +const zip = new JSZip(); +zip.file('mimetype', 'application/epub+zip', { compression: 'STORE' }); +zip.file( + 'META-INF/container.xml', + ` + + +`, +); + +const page = (title, body) => ` + + +${esc(title)} +${body} +`; + +const ids = chapters.map((_, i) => `ch${String(i + 1).padStart(4, '0')}`); +chapters.forEach((c, i) => { + zip.file( + `OEBPS/${ids[i]}.xhtml`, + page(c.name, `

${esc(c.name)}

\n${bodies[i]}`), + ); +}); + +const toc = chapters + .map((c, i) => `
  • ${esc(c.name)}
    • `) + .join('\n'); +zip.file( + 'OEBPS/nav.xhtml', + page('Contents', ``), +); + +let cover = ''; +if (novel.cover && /^https?:/.test(novel.cover)) { + try { + const res = await fetch(novel.cover, { headers: { Referer: plugin.site } }); + const type = (res.headers.get('content-type') || '').split(';')[0]; + const ext = { + 'image/jpeg': 'jpg', + 'image/png': 'png', + 'image/webp': 'webp', + 'image/gif': 'gif', + }[type]; + if (res.ok && ext) { + zip.file(`OEBPS/cover.${ext}`, Buffer.from(await res.arrayBuffer())); + cover = ``; + } + } catch { + // A missing cover is not worth failing the book for. + } +} + +const uid = `urn:lnreader:${plugin.id}:${novelPath}`; +const subjects = (novel.genres || '') + .split(',') + .map(g => g.trim()) + .filter(Boolean) + .map(g => `${esc(g)}`) + .join('\n '); +zip.file( + 'OEBPS/content.opf', + ` + + + ${esc(uid)} + ${esc(novel.name)} + en + ${novel.author ? `${esc(novel.author)}` : ''} + ${novel.summary ? `${esc(novel.summary)}` : ''} + ${esc(plugin.name)} + ${esc(plugin.resolveUrl ? plugin.resolveUrl(novelPath, true) : plugin.site)} + ${subjects} + ${new Date().toISOString().replace(/\.\d+Z$/, 'Z')} + + + + ${cover} + ${ids.map(id => ``).join('\n ')} + + + ${ids.map(id => ``).join('\n ')} + +`, +); + +fs.mkdirSync(outDir, { recursive: true }); +const file = path.join( + outDir, + `${novel.name.replace(/[/\\:*?"<>|]/g, '-')}.epub`, +); +fs.writeFileSync( + file, + await zip.generateAsync({ + type: 'nodebuffer', + compression: 'DEFLATE', + mimeType: 'application/epub+zip', + }), +); +print(file); +process.exit(0); diff --git a/scripts/find-novel.js b/scripts/find-novel.js index 42bada0..3b4f01a 100644 --- a/scripts/find-novel.js +++ b/scripts/find-novel.js @@ -1,20 +1,8 @@ // Search every plugin of the given languages for a title and list the sources // that carry it. Usage: node scripts/find-novel.js "" [lang...] // Languages are plugin folder names (default: english multi). -import * as esbuild from 'esbuild'; -import { execFileSync } from 'child_process'; -import { createRequire } from 'module'; -import { fileURLToPath } from 'url'; -import path, { dirname } from 'path'; -import fs from 'fs'; -import os from 'os'; +import { loadPlugins, print } from './load-plugins.js'; -const require = createRequire(import.meta.url); -// The shared fetch helper logs every request with its headers (credentials -// included); keep that out of the output. Results go to stdout explicitly. -const print = (...args) => process.stdout.write(args.join(' ') + '\n'); -console.log = () => {}; -const ROOT = path.resolve(dirname(fileURLToPath(import.meta.url)), '..'); const TIMEOUT_MS = 20_000; const CONCURRENCY = 16; const PER_SOURCE = 3; @@ -24,103 +12,68 @@ if (!term) { console.error('Usage: node scripts/find-novel.js "<title>" [lang...]'); process.exit(2); } -const langs = langArgs.length ? langArgs : ['english', 'multi']; -// Multisrc plugins (Madara, NovelFire, ...) are generated, not committed. -execFileSync('node', ['plugins/multisrc/generate-multisrc-plugins.js'], { - cwd: ROOT, - stdio: 'ignore', -}); +const plugins = await loadPlugins( + langArgs.length ? langArgs : ['english', 'multi'], +); +process.stderr.write(`Searching ${plugins.length} sources for "${term}"...\n`); -const entryPoints = langs.flatMap(lang => { - const dir = path.join(ROOT, 'plugins', lang); - if (!fs.existsSync(dir)) throw new Error(`No plugin folder: plugins/${lang}`); - return fs - .readdirSync(dir) - .filter(f => f.endsWith('.ts')) - .map(f => path.join(dir, f)); -}); - -const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'find-novel-')); -try { - const result = await esbuild.build({ - entryPoints, - outdir, - bundle: true, - platform: 'node', - format: 'cjs', - target: 'node22', - outExtension: { '.js': '.cjs' }, - logLevel: 'silent', - alias: { - '@libs': path.join(ROOT, 'src/libs'), - '@': path.join(ROOT, 'src'), - }, - }); - if (result.errors.length) throw new Error('Plugin bundling failed'); - - const bundles = fs - .readdirSync(outdir, { recursive: true }) - .filter(f => f.endsWith('.cjs')) - .map(f => path.join(outdir, f)); - process.stderr.write( - `Searching ${bundles.length} sources for "${term}"...\n`, - ); - - const hits = []; - let failed = 0; - const queue = [...bundles]; - const worker = async () => { - for (let file; (file = queue.shift());) { - let plugin; - try { - const mod = require(file); - plugin = mod.default ?? mod; - const found = await Promise.race([ - plugin.searchNovels(term, 1), - new Promise((_, reject) => - setTimeout(() => reject(new Error('timeout')), TIMEOUT_MS), - ), - ]); - for (const novel of (found || []).slice(0, PER_SOURCE)) { - const url = plugin.resolveUrl - ? plugin.resolveUrl(novel.path, true) - : new URL(novel.path, plugin.site).href; - hits.push({ source: plugin.name, name: novel.name, url }); - } - } catch { - failed++; +const hits = []; +let failed = 0; +const queue = [...plugins]; +const worker = async () => { + for (let plugin; (plugin = queue.shift());) { + try { + const found = await Promise.race([ + plugin.searchNovels(term, 1), + new Promise((_, reject) => + setTimeout(() => reject(new Error('timeout')), TIMEOUT_MS), + ), + ]); + for (const novel of (found || []).slice(0, PER_SOURCE)) { + const url = plugin.resolveUrl + ? plugin.resolveUrl(novel.path, true) + : new URL(novel.path, plugin.site).href; + hits.push({ + source: plugin.name, + id: plugin.id, + name: novel.name, + path: novel.path, + url, + }); } + } catch { + failed++; } - }; - await Promise.all(Array.from({ length: CONCURRENCY }, worker)); - - // Keep titles containing every word of the query (quotes/punctuation ignored). - const norm = t => - t - .toLowerCase() - .replace(/[’']/g, '') - .replace(/[^a-z0-9]+/g, ' ') - .trim(); - const words = norm(term).split(' '); - const exact = norm(term); - const matches = hits.filter(h => - words.every(w => norm(h.name).split(' ').includes(w)), - ); - const rank = h => (norm(h.name) === exact ? 0 : 1); - matches.sort((a, b) => rank(a) - rank(b) || a.source.localeCompare(b.source)); - - for (const h of matches) { - print( - `${rank(h) === 0 ? '*' : ' '} ${h.source.padEnd(28)} ${h.name}\n ${''.padEnd(28)} ${h.url}`, - ); } - process.stderr.write( - `${matches.length} matches (* = exact title), ${hits.length - matches.length} loose results hidden; ` + - `${bundles.length - failed}/${bundles.length} sources answered.\n`, +}; +await Promise.all(Array.from({ length: CONCURRENCY }, worker)); + +// Keep titles containing every word of the query (quotes/punctuation ignored). +const norm = t => + t + .toLowerCase() + .replace(/[’']/g, '') + .replace(/[^a-z0-9]+/g, ' ') + .trim(); +const words = norm(term).split(' '); +const exact = norm(term); +const matches = hits.filter(h => + words.every(w => norm(h.name).split(' ').includes(w)), +); +const rank = h => (norm(h.name) === exact ? 0 : 1); +matches.sort((a, b) => rank(a) - rank(b) || a.source.localeCompare(b.source)); + +for (const h of matches) { + print( + `${rank(h) === 0 ? '*' : ' '} ${h.source.padEnd(28)} ${h.name}\n` + + ` ${''.padEnd(28)} ${h.url}\n` + + ` ${''.padEnd(28)} novel fetch ${h.id} ${h.path}`, ); -} finally { - fs.rmSync(outdir, { recursive: true, force: true }); } +process.stderr.write( + `${matches.length} matches (* = exact title), ${hits.length - matches.length} loose results hidden; ` + + `${plugins.length - failed}/${plugins.length} sources answered.\n`, +); // Keep-alive sockets can hold the event loop open; results are already printed. process.exit(0); diff --git a/scripts/load-plugins.js b/scripts/load-plugins.js new file mode 100644 index 0000000..d93d090 --- /dev/null +++ b/scripts/load-plugins.js @@ -0,0 +1,68 @@ +// Load compiled plugin instances outside the app, for the novel CLI scripts. +import * as esbuild from 'esbuild'; +import { execFileSync } from 'child_process'; +import { createRequire } from 'module'; +import { fileURLToPath } from 'url'; +import path, { dirname } from 'path'; +import fs from 'fs'; +import os from 'os'; + +const require = createRequire(import.meta.url); +export const ROOT = path.resolve(dirname(fileURLToPath(import.meta.url)), '..'); + +// The shared fetch helper logs every request with its headers (credentials +// included). Callers print their output explicitly with `print`. +export const print = (...args) => process.stdout.write(args.join(' ') + '\n'); +console.log = () => undefined; + +/** Bundle and instantiate every plugin in the given language folders. */ +export async function loadPlugins(langs) { + // Multisrc plugins (Madara, NovelFire, ...) are generated, not committed. + execFileSync('node', ['plugins/multisrc/generate-multisrc-plugins.js'], { + cwd: ROOT, + stdio: 'ignore', + }); + + const entryPoints = langs.flatMap(lang => { + const dir = path.join(ROOT, 'plugins', lang); + if (!fs.existsSync(dir)) + throw new Error(`No plugin folder: plugins/${lang}`); + return fs + .readdirSync(dir) + .filter(f => f.endsWith('.ts')) + .map(f => path.join(dir, f)); + }); + + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-plugins-')); + try { + const result = await esbuild.build({ + entryPoints, + outdir, + bundle: true, + platform: 'node', + format: 'cjs', + target: 'node22', + outExtension: { '.js': '.cjs' }, + logLevel: 'silent', + alias: { + '@libs': path.join(ROOT, 'src/libs'), + '@': path.join(ROOT, 'src'), + }, + }); + if (result.errors.length) throw new Error('Plugin bundling failed'); + + const plugins = []; + for (const file of fs.readdirSync(outdir, { recursive: true })) { + if (!file.endsWith('.cjs')) continue; + try { + const mod = require(path.join(outdir, file)); + plugins.push(mod.default ?? mod); + } catch { + // A plugin that throws on load is skipped, like a broken source in the app. + } + } + return plugins; + } finally { + fs.rmSync(outdir, { recursive: true, force: true }); + } +}