lnreader-plugins/scripts/fetch-novel.js

// Download a whole novel through its plugin and write it as an EPUB 3 file.
// Usage: node scripts/fetch-novel.js <plugin-id> <novel-path> <out-dir>
// Prints the written file path as the last stdout line.
import JSZip from 'jszip';
import fs from 'fs';
import path from 'path';
import { loadPlugin, print } from './load-plugins.js';
import { chapterPage, esc, page, toXhtml } from './epub.js';

// Be gentle with the source site: few parallel requests, retries with backoff.
const CONCURRENCY = 3;
const RETRIES = 4;

const [pluginId, novelPath, outDir] = process.argv.slice(2);
if (!pluginId || !novelPath || !outDir) {
  console.error(
    'Usage: node scripts/fetch-novel.js <plugin-id> <novel-path> <out-dir>',
  );
  process.exit(2);
}

const plugin = await loadPlugin(pluginId);

const retry = async (label, fn) => {
  for (let attempt = 1; ; attempt++) {
    try {
      return await fn();
    } catch (e) {
      if (attempt >= RETRIES) throw new Error(`${label}: ${e.message}`);
      await new Promise(r => setTimeout(r, 1000 * 2 ** attempt));
    }
  }
};

const novel = await retry('novel', () => plugin.parseNovel(novelPath));
// Plugins mark paid chapters with 🔒 and return only a preview for them. Keep the
// contiguous run of free chapters from the start: a teaser never becomes a chapter, and
// chapters freed later are appended in order by update-novels.js.
const all = novel.chapters || [];
const firstLocked = all.findIndex(c => c.name.includes('🔒'));
const chapters = firstLocked < 0 ? all : all.slice(0, firstLocked);
if (!chapters.length) throw new Error('Novel has no free chapters');
process.stderr.write(
  `${novel.name}: ${chapters.length} of ${all.length} chapters from ${plugin.name}` +
    (firstLocked < 0 ? '\n' : ' (the rest are locked)\n'),
);

const bodies = new Array(chapters.length);
let done = 0;
const queue = chapters.map((c, i) => i);
await Promise.all(
  Array.from({ length: CONCURRENCY }, async () => {
    for (let i; (i = queue.shift()) !== undefined;) {
      const html = await retry(chapters[i].name, () =>
        plugin.parseChapter(chapters[i].path),
      );
      bodies[i] = toXhtml(html);
      if (++done % 50 === 0 || done === chapters.length) {
        process.stderr.write(`  ${done}/${chapters.length}\n`);
      }
    }
  }),
);

const zip = new JSZip();
zip.file('mimetype', 'application/epub+zip', { compression: 'STORE' });
zip.file(
  'META-INF/container.xml',
  `<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
  <rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles>
</container>`,
);

const ids = chapters.map((_, i) => `ch${String(i + 1).padStart(4, '0')}`);
chapters.forEach((c, i) => {
  zip.file(`OEBPS/${ids[i]}.xhtml`, chapterPage(c.name, bodies[i]));
});

const toc = chapters
  .map((c, i) => `<li><a href="${ids[i]}.xhtml">${esc(c.name)}</a></li>`)
  .join('\n');
zip.file(
  'OEBPS/nav.xhtml',
  page('Contents', `<nav epub:type="toc"><ol>\n${toc}\n</ol></nav>`),
);

let cover = '';
if (novel.cover && /^https?:/.test(novel.cover)) {
  try {
    const res = await fetch(novel.cover, { headers: { Referer: plugin.site } });
    const type = (res.headers.get('content-type') || '').split(';')[0];
    const ext = {
      'image/jpeg': 'jpg',
      'image/png': 'png',
      'image/webp': 'webp',
      'image/gif': 'gif',
    }[type];
    if (res.ok && ext) {
      zip.file(`OEBPS/cover.${ext}`, Buffer.from(await res.arrayBuffer()));
      cover = `<item id="cover" href="cover.${ext}" media-type="${type}" properties="cover-image"/>`;
    }
  } catch {
    // A missing cover is not worth failing the book for.
  }
}

const uid = `urn:lnreader:${plugin.id}:${novelPath}`;
const subjects = (novel.genres || '')
  .split(',')
  .map(g => g.trim())
  .filter(Boolean)
  .map(g => `<dc:subject>${esc(g)}</dc:subject>`)
  .join('\n    ');
zip.file(
  'OEBPS/content.opf',
  `<?xml version="1.0" encoding="UTF-8"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid">
  <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
    <dc:identifier id="uid">${esc(uid)}</dc:identifier>
    <dc:title>${esc(novel.name)}</dc:title>
    <dc:language>en</dc:language>
    ${novel.author ? `<dc:creator>${esc(novel.author)}</dc:creator>` : ''}
    ${novel.summary ? `<dc:description>${esc(novel.summary)}</dc:description>` : ''}
    <dc:publisher>${esc(plugin.name)}</dc:publisher>
    <dc:source>${esc(plugin.resolveUrl ? plugin.resolveUrl(novelPath, true) : plugin.site)}</dc:source>
    ${subjects}
    <meta property="dcterms:modified">${new Date().toISOString().replace(/\.\d+Z$/, 'Z')}</meta>
  </metadata>
  <manifest>
    <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>
    ${cover}
    ${ids.map(id => `<item id="${id}" href="${id}.xhtml" media-type="application/xhtml+xml"/>`).join('\n    ')}
  </manifest>
  <spine>
    ${ids.map(id => `<itemref idref="${id}"/>`).join('\n    ')}
  </spine>
</package>`,
);

fs.mkdirSync(outDir, { recursive: true });
const file = path.join(
  outDir,
  `${novel.name.replace(/[/\\:*?"<>|]/g, '-')}.epub`,
);
fs.writeFileSync(
  file,
  await zip.generateAsync({
    type: 'nodebuffer',
    compression: 'DEFLATE',
    mimeType: 'application/epub+zip',
  }),
);
print(file);
process.exit(0);