// Novel requests over HTTP, for the "request a novel" page (served through the
// illustrator at /library). Searches and downloads run as short-lived child processes
// of find-novel.js / fetch-novel.js, so their memory (plugins carry their own
// libraries) is returned as soon as they finish; the server itself stays small.
// GET /health
// GET /find?q=
-> [{source, id, name, path, url, exact}] (cached 10 min)
// POST /fetch {id, path, name} -> job; one download at a time
// GET /jobs -> recent jobs, newest first
// GET /info?id=&path= -> {name, cover, author, status, genres, summary,
// chapters, free, latest} (cached 1 h, 3 at a time)
// GET /library -> titles already in the library
// Nightly (UPDATE_HOUR, local time) update-novels.js appends new free chapters.
// Env: LIBRARY (novels dir, writable), PORT, NOVEL_PLUGIN_CACHE, UPDATE_HOUR.
import { spawn } from 'child_process';
import fs from 'fs';
import http from 'http';
import os from 'os';
import path from 'path';
import { fileURLToPath } from 'url';
const HERE = path.dirname(fileURLToPath(import.meta.url));
const LIBRARY = process.env.LIBRARY || '/library';
const PORT = Number(process.env.PORT || 8090);
const UPDATE_HOUR = Number(process.env.UPDATE_HOUR || 4);
const FIND_TTL_MS = 10 * 60 * 1000;
const INFO_TTL_MS = 60 * 60 * 1000;
const INFO_CONCURRENCY = 3;
const MAX_JOBS = 50;
const found = new Map(); // query -> {at, results}
const infos = new Map(); // id\npath -> {at, info}
let infoRunning = 0;
const infoWaiting = [];
// At most INFO_CONCURRENCY detail lookups at once: each is a child process and a request
// to the source site.
async function info(id, novelPath) {
const cacheKey = `${id}\n${novelPath}`;
const hit = infos.get(cacheKey);
if (hit && Date.now() - hit.at < INFO_TTL_MS) return hit.info;
if (infoRunning >= INFO_CONCURRENCY)
await new Promise(resolve => infoWaiting.push(resolve));
infoRunning++;
try {
const { code, out, err } = await run('novel-info.js', [id, novelPath]);
if (code !== 0) throw new Error(errorLine(err, code));
const result = JSON.parse(out.trim().split('\n').pop());
infos.set(cacheKey, { at: Date.now(), info: result });
return result;
} finally {
infoRunning--;
infoWaiting.shift()?.();
}
}
const jobs = []; // newest first
let running = null;
function run(script, args) {
return new Promise(resolve => {
const child = spawn(process.execPath, [path.join(HERE, script), ...args], {
stdio: ['ignore', 'pipe', 'pipe'],
});
let out = '';
let err = '';
child.stdout.on('data', d => (out += d));
child.stderr.on('data', d => {
err += d;
if (err.length > 20000) err = err.slice(-10000);
});
child.on('close', code => resolve({ code, out, err }));
});
}
// Node ends a crash with its version line; report the line that names the error.
function errorLine(err, code) {
const lines = err.trim().split('\n').filter(Boolean);
return (lines.findLast(l => /error/i.test(l)) || lines[0] || `exit ${code}`)
.trim()
.slice(0, 200);
}
const safeName = s => s.replace(/[/\\:*?"<>|]/g, '-').trim();
// One folder per series, named after the book: Kavita groups by folder.
function place(file) {
const name = safeName(path.basename(file, '.epub'));
const dir = path.join(LIBRARY, name);
const target = path.join(dir, `${name}.epub`);
if (fs.existsSync(target)) throw new Error(`already in the library: ${name}`);
fs.mkdirSync(dir, { recursive: true, mode: 0o2775 });
fs.copyFileSync(file, target);
return name;
}
async function work() {
if (running) return;
running = jobs.findLast(j => j.status === 'queued');
if (!running) return;
const job = running;
job.status = 'running';
job.started = Date.now();
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-'));
try {
const { code, out, err } = await run('fetch-novel.js', [
job.id,
job.path,
tmp,
]);
const file = out.trim().split('\n').pop();
if (code !== 0 || !file || !fs.existsSync(file)) {
throw new Error(errorLine(err, code));
}
job.progress = err.trim().split('\n')[0] || '';
job.title = place(file);
job.status = 'done';
} catch (e) {
job.status = 'failed';
job.error = String(e.message || e).slice(0, 300);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
job.finished = Date.now();
running = null;
setImmediate(work);
}
}
async function nightly() {
const next = new Date();
next.setHours(UPDATE_HOUR, 17, 0, 0);
if (next <= new Date()) next.setDate(next.getDate() + 1);
setTimeout(async () => {
const { code, out, err } = await run('update-novels.js', [LIBRARY]);
process.stdout.write(
`nightly update (exit ${code}):\n${out}${err.slice(-2000)}\n`,
);
nightly();
}, next - new Date());
}
function send(res, code, body) {
const data = JSON.stringify(body);
res.writeHead(code, {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(data),
'Cache-Control': 'no-store',
});
res.end(data);
}
function readJson(req) {
return new Promise((resolve, reject) => {
let body = '';
req.on('data', d => {
body += d;
if (body.length > 10000) reject(new Error('body too large'));
});
req.on('end', () => {
try {
resolve(JSON.parse(body || '{}'));
} catch {
reject(new Error('invalid JSON'));
}
});
});
}
http
.createServer(async (req, res) => {
const url = new URL(req.url, 'http://localhost');
try {
if (req.method === 'GET' && url.pathname === '/health') {
return send(res, 200, { ok: true });
}
if (req.method === 'GET' && url.pathname === '/find') {
const q = (url.searchParams.get('q') || '').trim().slice(0, 200);
if (q.length < 2) return send(res, 400, { error: 'q is required' });
const hit = found.get(q.toLowerCase());
if (hit && Date.now() - hit.at < FIND_TTL_MS)
return send(res, 200, hit.results);
const { code, out, err } = await run('find-novel.js', ['--json', q]);
if (code !== 0) return send(res, 502, { error: errorLine(err, code) });
const results = JSON.parse(out.trim().split('\n').pop() || '[]');
found.set(q.toLowerCase(), { at: Date.now(), results });
return send(res, 200, results);
}
if (req.method === 'GET' && url.pathname === '/info') {
const id = url.searchParams.get('id') || '';
const novelPath = url.searchParams.get('path') || '';
if (!id || !novelPath)
return send(res, 400, { error: 'id and path are required' });
try {
return send(res, 200, await info(id, novelPath));
} catch (e) {
return send(res, 502, {
error: String(e.message || e).slice(0, 300),
});
}
}
if (req.method === 'GET' && url.pathname === '/library') {
const titles = fs.existsSync(LIBRARY)
? fs
.readdirSync(LIBRARY, { withFileTypes: true })
.filter(d => d.isDirectory())
.map(d => d.name)
: [];
return send(res, 200, titles);
}
if (req.method === 'GET' && url.pathname === '/jobs') {
return send(res, 200, jobs);
}
if (req.method === 'POST' && url.pathname === '/fetch') {
const { id, path: novelPath, name } = await readJson(req);
if (
typeof id !== 'string' ||
typeof novelPath !== 'string' ||
!id ||
!novelPath
)
return send(res, 400, { error: 'id and path are required' });
const same = jobs.find(
j =>
j.id === id &&
j.path === novelPath &&
['queued', 'running'].includes(j.status),
);
if (same) return send(res, 202, same);
const job = {
id: id.slice(0, 100),
path: novelPath.slice(0, 500),
name: String(name || novelPath).slice(0, 200),
status: 'queued',
queued: Date.now(),
};
jobs.unshift(job);
jobs.length = Math.min(jobs.length, MAX_JOBS);
setImmediate(work);
return send(res, 202, job);
}
send(res, 404, { error: 'not found' });
} catch (e) {
send(res, 400, { error: String(e.message || e) });
}
})
.listen(PORT, () => process.stdout.write(`novel requests on :${PORT}\n`));
nightly();