diff --git a/.gitignore b/.gitignore index b84d1cc..a58f81d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ broken-sites-report.json .claude/ .cursor/ .plugin-cache/ +request-jobs.json* diff --git a/scripts/fetch-novel.js b/scripts/fetch-novel.js index 56da531..b81236a 100644 --- a/scripts/fetch-novel.js +++ b/scripts/fetch-novel.js @@ -55,7 +55,8 @@ await Promise.all( plugin.parseChapter(chapters[i].path), ); bodies[i] = toXhtml(html); - if (++done % 50 === 0 || done === chapters.length) { + // Every 5 chapters: the request page shows this as live progress. + if (++done % 5 === 0 || done === chapters.length) { process.stderr.write(` ${done}/${chapters.length}\n`); } } diff --git a/scripts/request-server.js b/scripts/request-server.js index bb0cb3c..2a2f24a 100644 --- a/scripts/request-server.js +++ b/scripts/request-server.js @@ -52,10 +52,34 @@ async function info(id, novelPath) { infoWaiting.shift()?.(); } } -const jobs = []; // newest first +// Jobs persist in a small JSON file, so the page shows them after a restart; a job that +// was running when the fetcher stopped is queued again. +const JOBS_FILE = path.join( + process.env.STATE_DIR || path.join(HERE, '..'), + 'request-jobs.json', +); +const jobs = (() => { + try { + return JSON.parse(fs.readFileSync(JOBS_FILE, 'utf8')).map(j => + j.status === 'running' + ? { ...j, status: 'queued', done: 0, phase: 'waiting' } + : j, + ); + } catch { + return []; + } +})(); // newest first let running = null; +function saveJobs() { + try { + fs.writeFileSync(`${JOBS_FILE}.tmp`, JSON.stringify(jobs)); + fs.renameSync(`${JOBS_FILE}.tmp`, JOBS_FILE); + } catch (e) { + process.stderr.write(`could not save jobs: ${e.message}\n`); + } +} -function run(script, args) { +function run(script, args, onStderrLine) { return new Promise(resolve => { const child = spawn(process.execPath, [path.join(HERE, script), ...args], { stdio: ['ignore', 'pipe', 'pipe'], @@ -63,9 +87,14 @@ function run(script, args) { let out = ''; let err = ''; child.stdout.on('data', d => (out += d)); + let partial = ''; child.stderr.on('data', d => { err += d; if (err.length > 20000) err = err.slice(-10000); + if (!onStderrLine) return; + const lines = (partial + d).split('\n'); + partial = lines.pop(); + lines.forEach(onStderrLine); }); child.on('close', code => resolve({ code, out, err })); }); @@ -99,26 +128,45 @@ async function work() { const job = running; job.status = 'running'; job.started = Date.now(); + job.phase = 'reading the chapter list'; + saveJobs(); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-')); try { - const { code, out, err } = await run('fetch-novel.js', [ - job.id, - job.path, - tmp, - ]); + // fetch-novel.js prints ": N of M chapters…" and then " done/total". + const { code, out, err } = await run( + 'fetch-novel.js', + [job.id, job.path, tmp], + line => { + const counts = line.match(/^\s*(\d+)\/(\d+)\s*$/); + if (counts) { + job.done = Number(counts[1]); + job.total = Number(counts[2]); + job.phase = + job.done < job.total ? 'downloading chapters' : 'building the book'; + } else if (/ chapters from /.test(line)) { + job.progress = line.trim(); + job.total = Number((line.match(/: (\d+) of/) || [])[1]) || job.total; + job.done = 0; + job.phase = 'downloading chapters'; + } + saveJobs(); + }, + ); const file = out.trim().split('\n').pop(); if (code !== 0 || !file || !fs.existsSync(file)) { throw new Error(errorLine(err, code)); } - job.progress = err.trim().split('\n')[0] || ''; + job.phase = 'adding to the library'; job.title = place(file); job.status = 'done'; + job.phase = 'in Kavita'; } catch (e) { job.status = 'failed'; job.error = String(e.message || e).slice(0, 300); } finally { fs.rmSync(tmp, { recursive: true, force: true }); job.finished = Date.now(); + saveJobs(); running = null; setImmediate(work); } @@ -233,6 +281,8 @@ http }; jobs.unshift(job); jobs.length = Math.min(jobs.length, MAX_JOBS); + job.phase = running ? 'waiting for the download before it' : 'starting'; + saveJobs(); setImmediate(work); return send(res, 202, job); } @@ -244,3 +294,4 @@ http .listen(PORT, () => process.stdout.write(`novel requests on :${PORT}\n`)); nightly(); +setImmediate(work); // resume jobs queued before a restart