diff --git a/scripts/epub.js b/scripts/epub.js new file mode 100644 index 0000000..18becca --- /dev/null +++ b/scripts/epub.js @@ -0,0 +1,28 @@ +// EPUB pieces shared by fetch-novel.js (new books) and update-novels.js (appends). +import { load } from 'cheerio'; +import { stripPromo } from './clean-chapter.js'; + +export const esc = s => + String(s ?? '').replace( + /[&<>"]/g, + c => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c], + ); + +// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without +// scripts, remote resources (an EPUB must be self-contained) or promo lines. +export const toXhtml = html => { + const $ = load(html, null, false); + $('script, style, iframe, link, meta, img, video, audio, source').remove(); + stripPromo($); + return $.xml(); +}; + +export const page = (title, body) => ` + + +${esc(title)} +${body} +`; + +export const chapterPage = (title, body) => + page(title, `

${esc(title)}

\n${body}`); diff --git a/scripts/fetch-novel.js b/scripts/fetch-novel.js index 7921b82..56da531 100644 --- a/scripts/fetch-novel.js +++ b/scripts/fetch-novel.js @@ -2,11 +2,10 @@ // Usage: node scripts/fetch-novel.js // Prints the written file path as the last stdout line. import JSZip from 'jszip'; -import { load } from 'cheerio'; import fs from 'fs'; import path from 'path'; -import { loadPlugins, print, ROOT } from './load-plugins.js'; -import { stripPromo } from './clean-chapter.js'; +import { loadPlugin, print } from './load-plugins.js'; +import { chapterPage, esc, page, toXhtml } from './epub.js'; // Be gentle with the source site: few parallel requests, retries with backoff. const CONCURRENCY = 3; @@ -20,12 +19,7 @@ if (!pluginId || !novelPath || !outDir) { process.exit(2); } -const langs = fs - .readdirSync(path.join(ROOT, 'plugins'), { withFileTypes: true }) - .filter(d => d.isDirectory() && d.name !== 'multisrc') - .map(d => d.name); -const plugin = (await loadPlugins(langs)).find(p => p.id === pluginId); -if (!plugin) throw new Error(`No plugin with id "${pluginId}"`); +const plugin = await loadPlugin(pluginId); const retry = async (label, fn) => { for (let attempt = 1; ; attempt++) { @@ -39,27 +33,18 @@ const retry = async (label, fn) => { }; const novel = await retry('novel', () => plugin.parseNovel(novelPath)); -const chapters = novel.chapters || []; -if (!chapters.length) throw new Error('Novel has no chapters'); +// Plugins mark paid chapters with 🔒 and return only a preview for them. Keep the +// contiguous run of free chapters from the start: a teaser never becomes a chapter, and +// chapters freed later are appended in order by update-novels.js. +const all = novel.chapters || []; +const firstLocked = all.findIndex(c => c.name.includes('🔒')); +const chapters = firstLocked < 0 ? all : all.slice(0, firstLocked); +if (!chapters.length) throw new Error('Novel has no free chapters'); process.stderr.write( - `${novel.name}: ${chapters.length} chapters from ${plugin.name}\n`, + `${novel.name}: ${chapters.length} of ${all.length} chapters from ${plugin.name}` + + (firstLocked < 0 ? '\n' : ' (the rest are locked)\n'), ); -const esc = s => - String(s ?? '').replace( - /[&<>"]/g, - c => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c], - ); - -// Chapter HTML is tag soup; re-serialise it as well-formed XHTML without -// scripts, remote resources (an EPUB must be self-contained) or promo lines. -const toXhtml = html => { - const $ = load(html, null, false); - $('script, style, iframe, link, meta, img, video, audio, source').remove(); - stripPromo($); - return $.xml(); -}; - const bodies = new Array(chapters.length); let done = 0; const queue = chapters.map((c, i) => i); @@ -87,19 +72,9 @@ zip.file( `, ); -const page = (title, body) => ` - - -${esc(title)} -${body} -`; - const ids = chapters.map((_, i) => `ch${String(i + 1).padStart(4, '0')}`); chapters.forEach((c, i) => { - zip.file( - `OEBPS/${ids[i]}.xhtml`, - page(c.name, `

${esc(c.name)}

\n${bodies[i]}`), - ); + zip.file(`OEBPS/${ids[i]}.xhtml`, chapterPage(c.name, bodies[i])); }); const toc = chapters diff --git a/scripts/find-novel.js b/scripts/find-novel.js index 3b4f01a..73eee42 100644 --- a/scripts/find-novel.js +++ b/scripts/find-novel.js @@ -1,13 +1,16 @@ // Search every plugin of the given languages for a title and list the sources -// that carry it. Usage: node scripts/find-novel.js "" [lang...] -// Languages are plugin folder names (default: english multi). +// that carry it. Usage: node scripts/find-novel.js [--json] "<title>" [lang...] +// Languages are plugin folder names (default: english multi). --json prints one JSON +// array of {source, id, name, path, url, exact} for programs (the request page). import { loadPlugins, print } from './load-plugins.js'; const TIMEOUT_MS = 20_000; const CONCURRENCY = 16; const PER_SOURCE = 3; -const [term, ...langArgs] = process.argv.slice(2); +const args = process.argv.slice(2); +const asJson = args[0] === '--json'; +const [term, ...langArgs] = asJson ? args.slice(1) : args; if (!term) { console.error('Usage: node scripts/find-novel.js "<title>" [lang...]'); process.exit(2); @@ -64,6 +67,10 @@ const matches = hits.filter(h => const rank = h => (norm(h.name) === exact ? 0 : 1); matches.sort((a, b) => rank(a) - rank(b) || a.source.localeCompare(b.source)); +if (asJson) { + print(JSON.stringify(matches.map(h => ({ ...h, exact: rank(h) === 0 })))); + process.exit(0); +} for (const h of matches) { print( `${rank(h) === 0 ? '*' : ' '} ${h.source.padEnd(28)} ${h.name}\n` + diff --git a/scripts/load-plugins.js b/scripts/load-plugins.js index d93d090..1267474 100644 --- a/scripts/load-plugins.js +++ b/scripts/load-plugins.js @@ -1,6 +1,10 @@ // Load compiled plugin instances outside the app, for the novel CLI scripts. +// Bundling ~165 plugins takes ~15 s and each bundle carries its own libraries, so the +// bundle is built once per source revision and cached, with an id -> file index: +// fetching a novel then loads a single plugin instead of all of them. import * as esbuild from 'esbuild'; import { execFileSync } from 'child_process'; +import { createHash } from 'crypto'; import { createRequire } from 'module'; import { fileURLToPath } from 'url'; import path, { dirname } from 'path'; @@ -9,60 +13,114 @@ import os from 'os'; const require = createRequire(import.meta.url); export const ROOT = path.resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const CACHE = + process.env.NOVEL_PLUGIN_CACHE || + path.join(os.homedir(), '.cache', 'novel-plugins'); +export const LANGS = ['english', 'multi']; // The shared fetch helper logs every request with its headers (credentials // included). Callers print their output explicitly with `print`. export const print = (...args) => process.stdout.write(args.join(' ') + '\n'); console.log = () => undefined; -/** Bundle and instantiate every plugin in the given language folders. */ -export async function loadPlugins(langs) { +const git = args => + execFileSync('git', args, { cwd: ROOT, encoding: 'utf8' }).trim(); + +/** Cache key: the commit plus any uncommitted changes to plugins or libs. */ +function revision(langs) { + let head = 'nogit'; + let dirty = ''; + try { + head = git(['rev-parse', 'HEAD']); + dirty = git(['diff', 'HEAD', '--', 'plugins', 'src']); + } catch { + // Not a git checkout: the cache is keyed by language set only. + } + return createHash('sha256') + .update(`${head}\n${dirty}\n${langs.join(',')}`) + .digest('hex') + .slice(0, 16); +} + +/** Build (once per revision) and return the cache dir holding bundles + index.json. */ +async function bundle(langs) { + const dir = path.join(CACHE, revision(langs)); + if (fs.existsSync(path.join(dir, 'index.json'))) return dir; + // Multisrc plugins (Madara, NovelFire, ...) are generated, not committed. execFileSync('node', ['plugins/multisrc/generate-multisrc-plugins.js'], { cwd: ROOT, stdio: 'ignore', }); - const entryPoints = langs.flatMap(lang => { - const dir = path.join(ROOT, 'plugins', lang); - if (!fs.existsSync(dir)) + const src = path.join(ROOT, 'plugins', lang); + if (!fs.existsSync(src)) throw new Error(`No plugin folder: plugins/${lang}`); return fs - .readdirSync(dir) + .readdirSync(src) .filter(f => f.endsWith('.ts')) - .map(f => path.join(dir, f)); + .map(f => path.join(src, f)); }); - const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-plugins-')); - try { - const result = await esbuild.build({ - entryPoints, - outdir, - bundle: true, - platform: 'node', - format: 'cjs', - target: 'node22', - outExtension: { '.js': '.cjs' }, - logLevel: 'silent', - alias: { - '@libs': path.join(ROOT, 'src/libs'), - '@': path.join(ROOT, 'src'), - }, - }); - if (result.errors.length) throw new Error('Plugin bundling failed'); + const tmp = `${dir}.tmp-${process.pid}`; + fs.rmSync(tmp, { recursive: true, force: true }); + const result = await esbuild.build({ + entryPoints, + outdir: tmp, + bundle: true, + platform: 'node', + format: 'cjs', + target: 'node22', + outExtension: { '.js': '.cjs' }, + logLevel: 'silent', + alias: { + '@libs': path.join(ROOT, 'src/libs'), + '@': path.join(ROOT, 'src'), + }, + }); + if (result.errors.length) throw new Error('Plugin bundling failed'); - const plugins = []; - for (const file of fs.readdirSync(outdir, { recursive: true })) { - if (!file.endsWith('.cjs')) continue; - try { - const mod = require(path.join(outdir, file)); - plugins.push(mod.default ?? mod); - } catch { - // A plugin that throws on load is skipped, like a broken source in the app. - } + const index = {}; + for (const file of fs.readdirSync(tmp, { recursive: true })) { + if (!file.endsWith('.cjs')) continue; + try { + const mod = require(path.join(tmp, file)); + const plugin = mod.default ?? mod; + if (plugin?.id) index[plugin.id] = file; + } catch { + // A plugin that throws on load is skipped, like a broken source in the app. } - return plugins; - } finally { - fs.rmSync(outdir, { recursive: true, force: true }); } + fs.writeFileSync(path.join(tmp, 'index.json'), JSON.stringify(index)); + fs.rmSync(dir, { recursive: true, force: true }); + fs.renameSync(tmp, dir); + return dir; +} + +function instantiate(dir, file) { + const mod = require(path.join(dir, file)); + return mod.default ?? mod; +} + +/** Every plugin in the given language folders (for searching across sources). */ +export async function loadPlugins(langs = LANGS) { + const dir = await bundle(langs); + const index = JSON.parse(fs.readFileSync(path.join(dir, 'index.json'))); + const plugins = []; + for (const file of Object.values(index)) { + try { + plugins.push(instantiate(dir, file)); + } catch { + // Skipped, as at bundle time. + } + } + return plugins; +} + +/** One plugin by id, loading nothing else. */ +export async function loadPlugin(id, langs = LANGS) { + const dir = await bundle(langs); + const file = JSON.parse(fs.readFileSync(path.join(dir, 'index.json')))[id]; + if (!file) throw new Error(`No plugin with id "${id}"`); + return instantiate(dir, file); } diff --git a/scripts/update-novels.js b/scripts/update-novels.js new file mode 100644 index 0000000..d3e85df --- /dev/null +++ b/scripts/update-novels.js @@ -0,0 +1,113 @@ +// Append newly published (and newly free) chapters to EPUBs built by fetch-novel.js. +// Usage: node scripts/update-novels.js <library-dir> +// Only EPUBs whose identifier is urn:lnreader:<plugin>:<path> are touched. Chapters are +// appended at the end, so existing positions (Kavita pages, reading progress, +// illustration anchors) never move. Prints one line per updated novel. +import JSZip from 'jszip'; +import { load } from 'cheerio'; +import fs from 'fs'; +import path from 'path'; +import { loadPlugin, print } from './load-plugins.js'; +import { chapterPage, esc, toXhtml } from './epub.js'; + +const CONCURRENCY = 3; +const RETRIES = 4; + +const [library] = process.argv.slice(2); +if (!library) { + console.error('Usage: node scripts/update-novels.js <library-dir>'); + process.exit(2); +} + +const retry = async (label, fn) => { + for (let attempt = 1; ; attempt++) { + try { + return await fn(); + } catch (e) { + if (attempt >= RETRIES) throw new Error(`${label}: ${e.message}`); + await new Promise(r => setTimeout(r, 1000 * 2 ** attempt)); + } + } +}; + +async function update(file) { + const zip = await JSZip.loadAsync(fs.readFileSync(file)); + const opfPath = 'OEBPS/content.opf'; + if (!zip.file(opfPath)) return; + const opf = load(await zip.file(opfPath).async('string'), { xml: true }); + const uid = opf('dc\\:identifier').first().text(); + const m = uid.match(/^urn:lnreader:([^:]+):(.+)$/); + if (!m) return; + const [, pluginId, novelPath] = m; + const have = opf('spine itemref').length; + + const plugin = await loadPlugin(pluginId); + const novel = await retry('novel', () => plugin.parseNovel(novelPath)); + const all = novel.chapters || []; + const firstLocked = all.findIndex(c => c.name.includes('🔒')); + const free = firstLocked < 0 ? all : all.slice(0, firstLocked); + if (free.length <= have) return; + + const added = free.slice(have); + const bodies = new Array(added.length); + const queue = added.map((_, i) => i); + await Promise.all( + Array.from({ length: CONCURRENCY }, async () => { + for (let i; (i = queue.shift()) !== undefined;) { + bodies[i] = toXhtml( + await retry(added[i].name, () => plugin.parseChapter(added[i].path)), + ); + } + }), + ); + + const nav = load(await zip.file('OEBPS/nav.xhtml').async('string'), { + xml: true, + }); + added.forEach((c, i) => { + const id = `ch${String(have + i + 1).padStart(4, '0')}`; + zip.file(`OEBPS/${id}.xhtml`, chapterPage(c.name, bodies[i])); + opf('manifest').append( + `<item id="${id}" href="${id}.xhtml" media-type="application/xhtml+xml"/>`, + ); + opf('spine').append(`<itemref idref="${id}"/>`); + nav('nav ol') + .first() + .append(`<li><a href="${id}.xhtml">${esc(c.name)}</a></li>`); + }); + opf('meta[property="dcterms:modified"]').text( + new Date().toISOString().replace(/\.\d+Z$/, 'Z'), + ); + zip.file(opfPath, opf.xml()); + zip.file('OEBPS/nav.xhtml', nav.xml()); + // EPUB requires mimetype stored uncompressed; loaded entries lose that option. + zip.file('mimetype', 'application/epub+zip', { compression: 'STORE' }); + const tmp = `${file}.tmp`; + fs.writeFileSync( + tmp, + await zip.generateAsync({ + type: 'nodebuffer', + compression: 'DEFLATE', + mimeType: 'application/epub+zip', + }), + ); + fs.renameSync(tmp, file); + print( + `${path.basename(file)}: +${added.length} chapters (${have} -> ${free.length})`, + ); +} + +const epubs = fs + .readdirSync(library, { recursive: true }) + .filter(f => f.endsWith('.epub')) + .map(f => path.join(library, f)); +let failed = 0; +for (const file of epubs) { + try { + await update(file); + } catch (e) { + failed++; + process.stderr.write(`${path.basename(file)}: ${e.message}\n`); + } +} +process.exit(failed ? 1 : 0);