// Download a whole novel through its plugin and write it as an EPUB 3 file. // Usage: node scripts/fetch-novel.js // Prints the written file path as the last stdout line. import JSZip from 'jszip'; import { load } from 'cheerio'; import fs from 'fs'; import path from 'path'; import { loadPlugins, print, ROOT } from './load-plugins.js'; // Be gentle with the source site: few parallel requests, retries with backoff. const CONCURRENCY = 3; const RETRIES = 4; const [pluginId, novelPath, outDir] = process.argv.slice(2); if (!pluginId || !novelPath || !outDir) { console.error( 'Usage: node scripts/fetch-novel.js ', ); process.exit(2); } const langs = fs .readdirSync(path.join(ROOT, 'plugins'), { withFileTypes: true }) .filter(d => d.isDirectory() && d.name !== 'multisrc') .map(d => d.name); const plugin = (await loadPlugins(langs)).find(p => p.id === pluginId); if (!plugin) throw new Error(`No plugin with id "${pluginId}"`); const retry = async (label, fn) => { for (let attempt = 1; ; attempt++) { try { return await fn(); } catch (e) { if (attempt >= RETRIES) throw new Error(`${label}: ${e.message}`); await new Promise(r => setTimeout(r, 1000 * 2 ** attempt)); } } }; const novel = await retry('novel', () => plugin.parseNovel(novelPath)); const chapters = novel.chapters || []; if (!chapters.length) throw new Error('Novel has no chapters'); process.stderr.write( `${novel.name}: ${chapters.length} chapters from ${plugin.name}\n`, ); const esc = s => String(s ?? '').replace( /[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c], ); // Chapter HTML is tag soup; re-serialise it as well-formed XHTML without // scripts or remote resources (an EPUB must be self-contained). const toXhtml = html => { const $ = load(html, null, false); $('script, style, iframe, link, meta, img, video, audio, source').remove(); return $.xml(); }; const bodies = new Array(chapters.length); let done = 0; const queue = chapters.map((c, i) => i); await Promise.all( Array.from({ length: CONCURRENCY }, async () => { for (let i; (i = queue.shift()) !== undefined;) { const html = await retry(chapters[i].name, () => plugin.parseChapter(chapters[i].path), ); bodies[i] = toXhtml(html); if (++done % 50 === 0 || done === chapters.length) { process.stderr.write(` ${done}/${chapters.length}\n`); } } }), ); const zip = new JSZip(); zip.file('mimetype', 'application/epub+zip', { compression: 'STORE' }); zip.file( 'META-INF/container.xml', ` `, ); const page = (title, body) => ` ${esc(title)} ${body} `; const ids = chapters.map((_, i) => `ch${String(i + 1).padStart(4, '0')}`); chapters.forEach((c, i) => { zip.file( `OEBPS/${ids[i]}.xhtml`, page(c.name, `

${esc(c.name)}

\n${bodies[i]}`), ); }); const toc = chapters .map((c, i) => `
  • ${esc(c.name)}
  • `) .join('\n'); zip.file( 'OEBPS/nav.xhtml', page('Contents', ``), ); let cover = ''; if (novel.cover && /^https?:/.test(novel.cover)) { try { const res = await fetch(novel.cover, { headers: { Referer: plugin.site } }); const type = (res.headers.get('content-type') || '').split(';')[0]; const ext = { 'image/jpeg': 'jpg', 'image/png': 'png', 'image/webp': 'webp', 'image/gif': 'gif', }[type]; if (res.ok && ext) { zip.file(`OEBPS/cover.${ext}`, Buffer.from(await res.arrayBuffer())); cover = ``; } } catch { // A missing cover is not worth failing the book for. } } const uid = `urn:lnreader:${plugin.id}:${novelPath}`; const subjects = (novel.genres || '') .split(',') .map(g => g.trim()) .filter(Boolean) .map(g => `${esc(g)}`) .join('\n '); zip.file( 'OEBPS/content.opf', ` ${esc(uid)} ${esc(novel.name)} en ${novel.author ? `${esc(novel.author)}` : ''} ${novel.summary ? `${esc(novel.summary)}` : ''} ${esc(plugin.name)} ${esc(plugin.resolveUrl ? plugin.resolveUrl(novelPath, true) : plugin.site)} ${subjects} ${new Date().toISOString().replace(/\.\d+Z$/, 'Z')} ${cover} ${ids.map(id => ``).join('\n ')} ${ids.map(id => ``).join('\n ')} `, ); fs.mkdirSync(outDir, { recursive: true }); const file = path.join( outDir, `${novel.name.replace(/[/\\:*?"<>|]/g, '-')}.epub`, ); fs.writeFileSync( file, await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', mimeType: 'application/epub+zip', }), ); print(file); process.exit(0);