masi
241 lines
8.2 KiB
JavaScript
241 lines
8.2 KiB
JavaScript
// Novel requests over HTTP, for the "request a novel" page (served through the
|
|
// illustrator at /library). Searches and downloads run as short-lived child processes
|
|
// of find-novel.js / fetch-novel.js, so their memory (plugins carry their own
|
|
// libraries) is returned as soon as they finish; the server itself stays small.
|
|
// GET /health
|
|
// GET /find?q=<title> -> [{source, id, name, path, url, exact}] (cached 10 min)
|
|
// POST /fetch {id, path, name} -> job; one download at a time
|
|
// GET /jobs -> recent jobs, newest first
|
|
// GET /info?id=&path= -> {name, cover, author, status, genres, summary,
|
|
// chapters, free, latest} (cached 1 h, 3 at a time)
|
|
// GET /library -> titles already in the library
|
|
// Nightly (UPDATE_HOUR, local time) update-novels.js appends new free chapters.
|
|
// Env: LIBRARY (novels dir, writable), PORT, NOVEL_PLUGIN_CACHE, UPDATE_HOUR.
|
|
import { spawn } from 'child_process';
|
|
import fs from 'fs';
|
|
import http from 'http';
|
|
import os from 'os';
|
|
import path from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
|
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
const LIBRARY = process.env.LIBRARY || '/library';
|
|
const PORT = Number(process.env.PORT || 8090);
|
|
const UPDATE_HOUR = Number(process.env.UPDATE_HOUR || 4);
|
|
const FIND_TTL_MS = 10 * 60 * 1000;
|
|
const INFO_TTL_MS = 60 * 60 * 1000;
|
|
const INFO_CONCURRENCY = 3;
|
|
const MAX_JOBS = 50;
|
|
|
|
const found = new Map(); // query -> {at, results}
|
|
const infos = new Map(); // id\npath -> {at, info}
|
|
let infoRunning = 0;
|
|
const infoWaiting = [];
|
|
|
|
// At most INFO_CONCURRENCY detail lookups at once: each is a child process and a request
|
|
// to the source site.
|
|
async function info(id, novelPath) {
|
|
const cacheKey = `${id}\n${novelPath}`;
|
|
const hit = infos.get(cacheKey);
|
|
if (hit && Date.now() - hit.at < INFO_TTL_MS) return hit.info;
|
|
if (infoRunning >= INFO_CONCURRENCY)
|
|
await new Promise(resolve => infoWaiting.push(resolve));
|
|
infoRunning++;
|
|
try {
|
|
const { code, out, err } = await run('novel-info.js', [id, novelPath]);
|
|
if (code !== 0)
|
|
throw new Error(err.trim().split('\n').pop() || `exit ${code}`);
|
|
const result = JSON.parse(out.trim().split('\n').pop());
|
|
infos.set(cacheKey, { at: Date.now(), info: result });
|
|
return result;
|
|
} finally {
|
|
infoRunning--;
|
|
infoWaiting.shift()?.();
|
|
}
|
|
}
|
|
const jobs = []; // newest first
|
|
let running = null;
|
|
|
|
function run(script, args) {
|
|
return new Promise(resolve => {
|
|
const child = spawn(process.execPath, [path.join(HERE, script), ...args], {
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
let out = '';
|
|
let err = '';
|
|
child.stdout.on('data', d => (out += d));
|
|
child.stderr.on('data', d => {
|
|
err += d;
|
|
if (err.length > 20000) err = err.slice(-10000);
|
|
});
|
|
child.on('close', code => resolve({ code, out, err }));
|
|
});
|
|
}
|
|
|
|
const safeName = s => s.replace(/[/\\:*?"<>|]/g, '-').trim();
|
|
|
|
// One folder per series, named after the book: Kavita groups by folder.
|
|
function place(file) {
|
|
const name = safeName(path.basename(file, '.epub'));
|
|
const dir = path.join(LIBRARY, name);
|
|
const target = path.join(dir, `${name}.epub`);
|
|
if (fs.existsSync(target)) throw new Error(`already in the library: ${name}`);
|
|
fs.mkdirSync(dir, { recursive: true, mode: 0o2775 });
|
|
fs.copyFileSync(file, target);
|
|
return name;
|
|
}
|
|
|
|
async function work() {
|
|
if (running) return;
|
|
running = jobs.findLast(j => j.status === 'queued');
|
|
if (!running) return;
|
|
const job = running;
|
|
job.status = 'running';
|
|
job.started = Date.now();
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-'));
|
|
try {
|
|
const { code, out, err } = await run('fetch-novel.js', [
|
|
job.id,
|
|
job.path,
|
|
tmp,
|
|
]);
|
|
const file = out.trim().split('\n').pop();
|
|
if (code !== 0 || !file || !fs.existsSync(file)) {
|
|
throw new Error(err.trim().split('\n').pop() || `exit ${code}`);
|
|
}
|
|
job.progress = err.trim().split('\n')[0] || '';
|
|
job.title = place(file);
|
|
job.status = 'done';
|
|
} catch (e) {
|
|
job.status = 'failed';
|
|
job.error = String(e.message || e).slice(0, 300);
|
|
} finally {
|
|
fs.rmSync(tmp, { recursive: true, force: true });
|
|
job.finished = Date.now();
|
|
running = null;
|
|
setImmediate(work);
|
|
}
|
|
}
|
|
|
|
async function nightly() {
|
|
const next = new Date();
|
|
next.setHours(UPDATE_HOUR, 17, 0, 0);
|
|
if (next <= new Date()) next.setDate(next.getDate() + 1);
|
|
setTimeout(async () => {
|
|
const { code, out, err } = await run('update-novels.js', [LIBRARY]);
|
|
process.stdout.write(
|
|
`nightly update (exit ${code}):\n${out}${err.slice(-2000)}\n`,
|
|
);
|
|
nightly();
|
|
}, next - new Date());
|
|
}
|
|
|
|
function send(res, code, body) {
|
|
const data = JSON.stringify(body);
|
|
res.writeHead(code, {
|
|
'Content-Type': 'application/json',
|
|
'Content-Length': Buffer.byteLength(data),
|
|
'Cache-Control': 'no-store',
|
|
});
|
|
res.end(data);
|
|
}
|
|
|
|
function readJson(req) {
|
|
return new Promise((resolve, reject) => {
|
|
let body = '';
|
|
req.on('data', d => {
|
|
body += d;
|
|
if (body.length > 10000) reject(new Error('body too large'));
|
|
});
|
|
req.on('end', () => {
|
|
try {
|
|
resolve(JSON.parse(body || '{}'));
|
|
} catch {
|
|
reject(new Error('invalid JSON'));
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
http
|
|
.createServer(async (req, res) => {
|
|
const url = new URL(req.url, 'http://localhost');
|
|
try {
|
|
if (req.method === 'GET' && url.pathname === '/health') {
|
|
return send(res, 200, { ok: true });
|
|
}
|
|
if (req.method === 'GET' && url.pathname === '/find') {
|
|
const q = (url.searchParams.get('q') || '').trim().slice(0, 200);
|
|
if (q.length < 2) return send(res, 400, { error: 'q is required' });
|
|
const hit = found.get(q.toLowerCase());
|
|
if (hit && Date.now() - hit.at < FIND_TTL_MS)
|
|
return send(res, 200, hit.results);
|
|
const { code, out, err } = await run('find-novel.js', ['--json', q]);
|
|
if (code !== 0)
|
|
return send(res, 502, { error: err.trim().split('\n').pop() });
|
|
const results = JSON.parse(out.trim().split('\n').pop() || '[]');
|
|
found.set(q.toLowerCase(), { at: Date.now(), results });
|
|
return send(res, 200, results);
|
|
}
|
|
if (req.method === 'GET' && url.pathname === '/info') {
|
|
const id = url.searchParams.get('id') || '';
|
|
const novelPath = url.searchParams.get('path') || '';
|
|
if (!id || !novelPath)
|
|
return send(res, 400, { error: 'id and path are required' });
|
|
try {
|
|
return send(res, 200, await info(id, novelPath));
|
|
} catch (e) {
|
|
return send(res, 502, {
|
|
error: String(e.message || e).slice(0, 300),
|
|
});
|
|
}
|
|
}
|
|
if (req.method === 'GET' && url.pathname === '/library') {
|
|
const titles = fs.existsSync(LIBRARY)
|
|
? fs
|
|
.readdirSync(LIBRARY, { withFileTypes: true })
|
|
.filter(d => d.isDirectory())
|
|
.map(d => d.name)
|
|
: [];
|
|
return send(res, 200, titles);
|
|
}
|
|
if (req.method === 'GET' && url.pathname === '/jobs') {
|
|
return send(res, 200, jobs);
|
|
}
|
|
if (req.method === 'POST' && url.pathname === '/fetch') {
|
|
const { id, path: novelPath, name } = await readJson(req);
|
|
if (
|
|
typeof id !== 'string' ||
|
|
typeof novelPath !== 'string' ||
|
|
!id ||
|
|
!novelPath
|
|
)
|
|
return send(res, 400, { error: 'id and path are required' });
|
|
const same = jobs.find(
|
|
j =>
|
|
j.id === id &&
|
|
j.path === novelPath &&
|
|
['queued', 'running'].includes(j.status),
|
|
);
|
|
if (same) return send(res, 202, same);
|
|
const job = {
|
|
id: id.slice(0, 100),
|
|
path: novelPath.slice(0, 500),
|
|
name: String(name || novelPath).slice(0, 200),
|
|
status: 'queued',
|
|
queued: Date.now(),
|
|
};
|
|
jobs.unshift(job);
|
|
jobs.length = Math.min(jobs.length, MAX_JOBS);
|
|
setImmediate(work);
|
|
return send(res, 202, job);
|
|
}
|
|
send(res, 404, { error: 'not found' });
|
|
} catch (e) {
|
|
send(res, 400, { error: String(e.message || e) });
|
|
}
|
|
})
|
|
.listen(PORT, () => process.stdout.write(`novel requests on :${PORT}\n`));
|
|
|
|
nightly();
|