feat(scripts): live download progress and persisted request list

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitignore

@@ -20,3 +20,4 @@ broken-sites-report.json
 .claude/
 .cursor/
 .plugin-cache/
+request-jobs.json*

scripts/fetch-novel.js

@@ -55,7 +55,8 @@ await Promise.all(
         plugin.parseChapter(chapters[i].path),
       );
       bodies[i] = toXhtml(html);
-      if (++done % 50 === 0 || done === chapters.length) {
+      // Every 5 chapters: the request page shows this as live progress.
+      if (++done % 5 === 0 || done === chapters.length) {
         process.stderr.write(`  ${done}/${chapters.length}\n`);
       }
     }

scripts/request-server.js

@@ -52,10 +52,34 @@ async function info(id, novelPath) {
     infoWaiting.shift()?.();
   }
 }
-const jobs = []; // newest first
+// Jobs persist in a small JSON file, so the page shows them after a restart; a job that
+// was running when the fetcher stopped is queued again.
+const JOBS_FILE = path.join(
+  process.env.STATE_DIR || path.join(HERE, '..'),
+  'request-jobs.json',
+);
+const jobs = (() => {
+  try {
+    return JSON.parse(fs.readFileSync(JOBS_FILE, 'utf8')).map(j =>
+      j.status === 'running'
+        ? { ...j, status: 'queued', done: 0, phase: 'waiting' }
+        : j,
+    );
+  } catch {
+    return [];
+  }
+})(); // newest first
 let running = null;
+function saveJobs() {
+  try {
+    fs.writeFileSync(`${JOBS_FILE}.tmp`, JSON.stringify(jobs));
+    fs.renameSync(`${JOBS_FILE}.tmp`, JOBS_FILE);
+  } catch (e) {
+    process.stderr.write(`could not save jobs: ${e.message}\n`);
+  }
+}
 
-function run(script, args) {
+function run(script, args, onStderrLine) {
   return new Promise(resolve => {
     const child = spawn(process.execPath, [path.join(HERE, script), ...args], {
       stdio: ['ignore', 'pipe', 'pipe'],
@@ -63,9 +87,14 @@ function run(script, args) {
     let out = '';
     let err = '';
     child.stdout.on('data', d => (out += d));
+    let partial = '';
     child.stderr.on('data', d => {
       err += d;
       if (err.length > 20000) err = err.slice(-10000);
+      if (!onStderrLine) return;
+      const lines = (partial + d).split('\n');
+      partial = lines.pop();
+      lines.forEach(onStderrLine);
     });
     child.on('close', code => resolve({ code, out, err }));
   });
@@ -99,26 +128,45 @@ async function work() {
   const job = running;
   job.status = 'running';
   job.started = Date.now();
+  job.phase = 'reading the chapter list';
+  saveJobs();
   const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'novel-'));
   try {
-    const { code, out, err } = await run('fetch-novel.js', [
-      job.id,
-      job.path,
-      tmp,
-    ]);
+    // fetch-novel.js prints "<name>: N of M chapters…" and then "  done/total".
+    const { code, out, err } = await run(
+      'fetch-novel.js',
+      [job.id, job.path, tmp],
+      line => {
+        const counts = line.match(/^\s*(\d+)\/(\d+)\s*$/);
+        if (counts) {
+          job.done = Number(counts[1]);
+          job.total = Number(counts[2]);
+          job.phase =
+            job.done < job.total ? 'downloading chapters' : 'building the book';
+        } else if (/ chapters from /.test(line)) {
+          job.progress = line.trim();
+          job.total = Number((line.match(/: (\d+) of/) || [])[1]) || job.total;
+          job.done = 0;
+          job.phase = 'downloading chapters';
+        }
+        saveJobs();
+      },
+    );
     const file = out.trim().split('\n').pop();
     if (code !== 0 || !file || !fs.existsSync(file)) {
       throw new Error(errorLine(err, code));
     }
-    job.progress = err.trim().split('\n')[0] || '';
+    job.phase = 'adding to the library';
     job.title = place(file);
     job.status = 'done';
+    job.phase = 'in Kavita';
   } catch (e) {
     job.status = 'failed';
     job.error = String(e.message || e).slice(0, 300);
   } finally {
     fs.rmSync(tmp, { recursive: true, force: true });
     job.finished = Date.now();
+    saveJobs();
     running = null;
     setImmediate(work);
   }
@@ -233,6 +281,8 @@ http
         };
         jobs.unshift(job);
         jobs.length = Math.min(jobs.length, MAX_JOBS);
+        job.phase = running ? 'waiting for the download before it' : 'starting';
+        saveJobs();
         setImmediate(work);
         return send(res, 202, job);
       }
@@ -244,3 +294,4 @@ http
   .listen(PORT, () => process.stdout.write(`novel requests on :${PORT}\n`));
 
 nightly();
+setImmediate(work); // resume jobs queued before a restart