// Novel requests over HTTP, for the "request a novel" page (served through the // illustrator at /library). Searches and downloads run as short-lived child processes // of find-novel.js / fetch-novel.js, so their memory (plugins carry their own // libraries) is returned as soon as they finish; the server itself stays small. // GET /health // GET /find?q= -> [{source, id, name, path, url, exact}] (cached 10 min) // POST /fetch {id, path, name} -> job; one download at a time // GET /jobs -> recent jobs, newest first // GET /info?id=&path= -> {name, cover, author, status, genres, summary, // chapters, free, latest} (cached 1 h, 3 at a time) // GET /library -> titles already in the library // Nightly (UPDATE_HOUR, local time) update-novels.js appends new free chapters. // Env: LIBRARY (novels dir, writable), PORT, NOVEL_PLUGIN_CACHE, UPDATE_HOUR. import { spawn } from 'child_process'; import fs from 'fs'; import http from 'http'; import os from 'os'; import path from 'path'; import { fileURLToPath } from 'url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const LIBRARY = process.env.LIBRARY || '/library'; const PORT = Number(process.env.PORT || 8090); const UPDATE_HOUR = Number(process.env.UPDATE_HOUR || 4); const FIND_TTL_MS = 10 * 60 * 1000; const INFO_TTL_MS = 60 * 60 * 1000; const INFO_CONCURRENCY = 3; const MAX_JOBS = 50; const found = new Map(); // query -> {at, results} const infos = new Map(); // id\npath -> {at, info} let infoRunning = 0; const infoWaiting = []; // At most INFO_CONCURRENCY detail lookups at once: each is a child process and a request // to the source site. async function info(id, novelPath) { const cacheKey = `${id}\n${novelPath}`; const hit = infos.get(cacheKey); if (hit && Date.now() - hit.at < INFO_TTL_MS) return hit.info; if (infoRunning >= INFO_CONCURRENCY) await new Promise(resolve => infoWaiting.push(resolve)); infoRunning++; try { const { code, out, err } = await run('novel-info.js', [id, novelPath]); if (code !== 0) throw new Error(err.trim().split('\n').pop() || `exit ${code}`); const result = JSON.parse(out.trim().split('\n').pop()); infos.set(cacheKey, { at: Date.now(), info: result }); return result; } finally { infoRunning--; infoWaiting.shift()?.(); } } const jobs = []; // newest first let running = null; function run(script, args) { return new Promise(resolve => { const child = spawn(process.execPath, [path.join(HERE, script), ...args], { stdio: ['ignore', 'pipe', 'pipe'], }); let out = ''; let err = ''; child.stdout.on('data', d => (out += d)); child.stderr.on('data', d => { err += d; if (err.length > 20000) err = err.slice(-10000); }); child.on('close', code => resolve({ code, out, err })); }); } const safeName = s => s.replace(/[/\\:*?"<>|]/g, '-').trim(); // One folder per series, named after the book: Kavita groups by folder. function place(file) { const name = safeName(path.basename(file, '.epub')); const dir = path.join(LIBRARY, name); const target = path.join(dir, `${name}.epub`); if (fs.existsSync(target)) throw new Error(`already in the library: ${name}`); fs.mkdirSync(dir, { recursive: true, mode: 0o2775 }); fs.copyFileSync(file, target); return name; } async function work() { if (running) return; running = jobs.findLast(j => j.status === 'queued'); if (!running) return; const job = running; job.status = 'running'; job.started = Date.now(); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-')); try { const { code, out, err } = await run('fetch-novel.js', [ job.id, job.path, tmp, ]); const file = out.trim().split('\n').pop(); if (code !== 0 || !file || !fs.existsSync(file)) { throw new Error(err.trim().split('\n').pop() || `exit ${code}`); } job.progress = err.trim().split('\n')[0] || ''; job.title = place(file); job.status = 'done'; } catch (e) { job.status = 'failed'; job.error = String(e.message || e).slice(0, 300); } finally { fs.rmSync(tmp, { recursive: true, force: true }); job.finished = Date.now(); running = null; setImmediate(work); } } async function nightly() { const next = new Date(); next.setHours(UPDATE_HOUR, 17, 0, 0); if (next <= new Date()) next.setDate(next.getDate() + 1); setTimeout(async () => { const { code, out, err } = await run('update-novels.js', [LIBRARY]); process.stdout.write( `nightly update (exit ${code}):\n${out}${err.slice(-2000)}\n`, ); nightly(); }, next - new Date()); } function send(res, code, body) { const data = JSON.stringify(body); res.writeHead(code, { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), 'Cache-Control': 'no-store', }); res.end(data); } function readJson(req) { return new Promise((resolve, reject) => { let body = ''; req.on('data', d => { body += d; if (body.length > 10000) reject(new Error('body too large')); }); req.on('end', () => { try { resolve(JSON.parse(body || '{}')); } catch { reject(new Error('invalid JSON')); } }); }); } http .createServer(async (req, res) => { const url = new URL(req.url, 'http://localhost'); try { if (req.method === 'GET' && url.pathname === '/health') { return send(res, 200, { ok: true }); } if (req.method === 'GET' && url.pathname === '/find') { const q = (url.searchParams.get('q') || '').trim().slice(0, 200); if (q.length < 2) return send(res, 400, { error: 'q is required' }); const hit = found.get(q.toLowerCase()); if (hit && Date.now() - hit.at < FIND_TTL_MS) return send(res, 200, hit.results); const { code, out, err } = await run('find-novel.js', ['--json', q]); if (code !== 0) return send(res, 502, { error: err.trim().split('\n').pop() }); const results = JSON.parse(out.trim().split('\n').pop() || '[]'); found.set(q.toLowerCase(), { at: Date.now(), results }); return send(res, 200, results); } if (req.method === 'GET' && url.pathname === '/info') { const id = url.searchParams.get('id') || ''; const novelPath = url.searchParams.get('path') || ''; if (!id || !novelPath) return send(res, 400, { error: 'id and path are required' }); try { return send(res, 200, await info(id, novelPath)); } catch (e) { return send(res, 502, { error: String(e.message || e).slice(0, 300), }); } } if (req.method === 'GET' && url.pathname === '/library') { const titles = fs.existsSync(LIBRARY) ? fs .readdirSync(LIBRARY, { withFileTypes: true }) .filter(d => d.isDirectory()) .map(d => d.name) : []; return send(res, 200, titles); } if (req.method === 'GET' && url.pathname === '/jobs') { return send(res, 200, jobs); } if (req.method === 'POST' && url.pathname === '/fetch') { const { id, path: novelPath, name } = await readJson(req); if ( typeof id !== 'string' || typeof novelPath !== 'string' || !id || !novelPath ) return send(res, 400, { error: 'id and path are required' }); const same = jobs.find( j => j.id === id && j.path === novelPath && ['queued', 'running'].includes(j.status), ); if (same) return send(res, 202, same); const job = { id: id.slice(0, 100), path: novelPath.slice(0, 500), name: String(name || novelPath).slice(0, 200), status: 'queued', queued: Date.now(), }; jobs.unshift(job); jobs.length = Math.min(jobs.length, MAX_JOBS); setImmediate(work); return send(res, 202, job); } send(res, 404, { error: 'not found' }); } catch (e) { send(res, 400, { error: String(e.message || e) }); } }) .listen(PORT, () => process.stdout.write(`novel requests on :${PORT}\n`)); nightly();