masi
975eb44707
Background GLM callers on the shared Ollama account slowed each scene call to ~2 min. Pre-reading and maintain.py now yield while illustrations are queued, passing a chapter drops queued work for earlier ones, and prompts carry only the active look's description (roster 90 KB -> 17 KB). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
408 lines
22 KiB
Python
408 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""novel-illustrator: illustrations, character cards and recaps for web novels read in
|
|
LNReader, in the style of each novel's cover.
|
|
|
|
A chapter's illustration: GLM (Ollama Cloud) picks a moment, writes the image prompt and
|
|
records what it learned (characters, their looks on the story timeline, glossary, a
|
|
summary); Qwen Image 2.1 renders it with the characters' portraits as references; GLM
|
|
checks the result and one re-roll replaces a defective image. See pipeline.py, store.py.
|
|
|
|
Endpoints (CORS open: the LNReader reader page runs on the source site's origin). "novel"
|
|
is always the canonical key from /v1/novels/resolve.
|
|
GET /reader.js script the LNReader code snippet loads
|
|
GET /gallery[?novel=] illustrations and character looks, up to the furthest chapter read
|
|
GET /v1/novels/resolve?key=&title= {"novel": canonical key}: one identity per book across sources
|
|
POST /v1/progress {novel, source, title, chapter, position}: chapter opened;
|
|
starts prefetch/pre-reading; -> {"recap_due": bool}
|
|
GET /v1/recap?novel=&position= {"recap": "..."|null} from the summaries of earlier chapters
|
|
GET /v1/settings?novel= {"every": N, "paused": bool}; every=0 is manual only
|
|
POST /v1/settings {novel, every, paused} -> settings
|
|
GET /v1/illustrations?novel=&chapter= status lookup, never starts work
|
|
POST /v1/illustrations {novel, chapter, position?, title?, chapter_title?, cover?,
|
|
genres?, text, regenerate?} -> job
|
|
GET /v1/illustrations/<id>[.webp] job status (with the placement anchor) / the image
|
|
DELETE /v1/illustrations/<id> cancel a queued job and skip the chapter for auto
|
|
GET /v1/codex?novel=&position= cards as of that chapter: characters (look there) and terms
|
|
POST /v1/characters {action: "redraw", novel, name, label, hint?} -> job
|
|
{action: "look", novel, name, label, position} -> card
|
|
{action: "merge", novel, name, into} -> {"merged": true}
|
|
GET /v1/portraits/<novel>/<look>.webp a portrait (WebP made from the PNG reference)
|
|
GET /v1/jobs/<id> status of a portrait job
|
|
GET /v1/styles?novel= {current, options: [{name, style, image}]}
|
|
POST /v1/styles {novel, action: "propose"} -> job | {novel, action: "choose", index}
|
|
GET /v1/styles/<file>.webp a style sample
|
|
GET /library request a novel (search sources, add to Kavita)
|
|
GET /v1/library/find?q=, /v1/library/jobs, POST /v1/library/fetch
|
|
proxied to the novel-fetcher service
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import threading
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
import gallery
|
|
import indexer
|
|
import pipeline
|
|
import store
|
|
from config import CHARACTERS, IMAGES, READER_JS, STYLES, key
|
|
|
|
MAX_BODY = 512 * 1024 # chapter text arrives in the request
|
|
FETCHER = os.environ.get("FETCHER_URL", "http://novel-fetcher:8090") # internal-only service
|
|
MAX_HINT = 300
|
|
MAX_UPLOAD = 6 * 1024 * 1024 # portrait uploads (base64 in JSON)
|
|
RECAP_AFTER = 2 * 24 * 3600 # offer a recap when a novel is reopened after two days
|
|
ID_RE = re.compile(r"^[0-9a-f]{32}$")
|
|
|
|
|
|
def text_field(req, field, required=False, limit=2000):
|
|
value = req.get(field)
|
|
if value is None or value == "":
|
|
if required:
|
|
raise ValueError(f"'{field}' is required")
|
|
return ""
|
|
if not isinstance(value, str) or (required and not value.strip()):
|
|
raise ValueError(f"'{field}' must be a string")
|
|
return value[:limit]
|
|
|
|
|
|
def position_field(req):
|
|
# 0-based chapter index in reading order; places story facts on the timeline.
|
|
position = req.get("position")
|
|
if position is not None and (type(position) is not int or position < 0):
|
|
raise ValueError("'position' must be a non-negative integer")
|
|
return position
|
|
|
|
|
|
def parse_illustration(req):
|
|
out = {field: text_field(req, field, required=True, limit=MAX_BODY) for field in ("novel", "chapter", "text")}
|
|
for field in ("title", "chapter_title", "cover", "genres"):
|
|
out[field] = text_field(req, field)
|
|
out["title"] = out["title"] or out["novel"]
|
|
out["chapter_title"] = out["chapter_title"] or out["chapter"]
|
|
out["position"] = position_field(req)
|
|
return out
|
|
|
|
|
|
def portrait_webp(novel_hash, look_file):
|
|
"""Portraits are stored as lossless PNG (they are Qwen references); phones get a small
|
|
WebP, remade whenever the PNG is redrawn."""
|
|
png = CHARACTERS / novel_hash / f"{look_file}.png"
|
|
if not png.exists():
|
|
return None
|
|
webp = png.with_suffix(".webp")
|
|
if not webp.exists() or webp.stat().st_mtime < png.stat().st_mtime:
|
|
tmp = png.parent / f"{look_file}.{threading.get_ident()}.tmp" # concurrent first requests
|
|
pipeline.to_webp(png, tmp)
|
|
tmp.replace(webp)
|
|
return webp
|
|
|
|
|
|
THUMB_WIDTHS = {"240", "480", "720"} # bounded, so thumbnails cannot fill the disk
|
|
|
|
|
|
def thumbnail(file, width):
|
|
"""A narrower WebP of an image for grids on phones, made once next to the original."""
|
|
if not file or width not in THUMB_WIDTHS:
|
|
return file
|
|
thumb = file.with_name(f"{file.stem}.w{width}.webp")
|
|
if not thumb.exists() or thumb.stat().st_mtime < file.stat().st_mtime:
|
|
from PIL import Image
|
|
tmp = file.with_name(f"{file.stem}.w{width}.{threading.get_ident()}.tmp")
|
|
with Image.open(file) as im:
|
|
im = im.convert("RGB")
|
|
im.thumbnail((int(width), int(width) * 2))
|
|
im.save(tmp, "WEBP", quality=80, method=4)
|
|
tmp.replace(thumb)
|
|
return thumb
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def send(self, code, body, content_type="application/json", cache=None):
|
|
data = body if isinstance(body, bytes) else json.dumps(body).encode()
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", content_type)
|
|
self.send_header("Content-Length", str(len(data)))
|
|
self.send_header("Access-Control-Allow-Origin", "*")
|
|
self.send_header("Cache-Control", cache or "no-store")
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
def do_OPTIONS(self):
|
|
self.send_response(204)
|
|
self.send_header("Access-Control-Allow-Origin", "*")
|
|
self.send_header("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
|
|
self.send_header("Access-Control-Allow-Headers", "Content-Type")
|
|
self.send_header("Access-Control-Max-Age", "86400")
|
|
self.end_headers()
|
|
|
|
def do_GET(self):
|
|
path, _, query = self.path.partition("?")
|
|
params = {k: v[0] for k, v in urllib.parse.parse_qs(query).items()}
|
|
novel = params.get("novel", "")
|
|
position = params.get("position", "")
|
|
if position and not position.isdigit():
|
|
return self.send(400, {"error": "position must be a non-negative integer"})
|
|
position = int(position) if position else None
|
|
|
|
if path == "/healthz":
|
|
return self.send(200, {"ok": True})
|
|
if path == "/reader.js":
|
|
return self.send(200, READER_JS.read_bytes(), "application/javascript", "max-age=300")
|
|
if path == "/library":
|
|
return self.send(200, gallery.library_page(), "text/html; charset=utf-8")
|
|
if path in ("/v1/library/find", "/v1/library/jobs", "/v1/library/info"):
|
|
return self.proxy(path.removeprefix("/v1/library"), query)
|
|
if path == "/v1/library/titles":
|
|
return self.proxy("/library")
|
|
if path == "/gallery":
|
|
html = gallery.wiki_page(novel) if novel else gallery.index()
|
|
return self.send(200, html, "text/html; charset=utf-8")
|
|
if path == "/v1/wiki":
|
|
if not novel:
|
|
return self.send(400, {"error": "novel is required"})
|
|
return self.send(200, gallery.wiki_data(novel))
|
|
if path == "/v1/novels/resolve":
|
|
if not params.get("key"):
|
|
return self.send(400, {"error": "key is required"})
|
|
return self.send(200, {"novel": store.resolve_novel(params["key"], params.get("title", ""))})
|
|
if path in ("/v1/settings", "/v1/codex", "/v1/recap", "/v1/illustrations") and not novel:
|
|
return self.send(400, {"error": "novel is required"})
|
|
if path == "/v1/settings":
|
|
return self.send(200, store.get_settings(novel))
|
|
if path == "/v1/codex":
|
|
# The reader's cards; the wiki calls store.codex with familiar terms included.
|
|
return self.send(200, store.codex(novel, position, familiar=False))
|
|
if path == "/v1/recap":
|
|
if position is None:
|
|
return self.send(400, {"error": "position is required"})
|
|
return self.send(200, {"recap": pipeline.recap(novel, position)})
|
|
if path == "/v1/illustrations":
|
|
# Status lookup by novel+chapter without starting any work.
|
|
if not params.get("chapter"):
|
|
return self.send(400, {"error": "chapter is required"})
|
|
return self.send(200, pipeline.chapter_status(novel, params["chapter"]))
|
|
m = re.fullmatch(r"/v1/portraits/([0-9a-f]{32})/([0-9a-f]{32}(?:-[0-9a-f]{8})?)\.webp", path)
|
|
if m:
|
|
file = thumbnail(portrait_webp(*m.groups()), params.get("w"))
|
|
if not file:
|
|
return self.send(404, {"error": "not found"})
|
|
return self.send(200, file.read_bytes(), "image/webp", "public, max-age=31536000, immutable")
|
|
if path == "/v1/styles":
|
|
if not novel:
|
|
return self.send(400, {"error": "novel is required"})
|
|
return self.send(200, pipeline.styles(novel))
|
|
m = re.fullmatch(r"/v1/styles/([0-9a-f]{32}-option\d)\.webp", path)
|
|
if m:
|
|
file = STYLES / f"{m.group(1)}.webp"
|
|
if not file.exists():
|
|
return self.send(404, {"error": "not found"})
|
|
file = thumbnail(file, params.get("w"))
|
|
return self.send(200, file.read_bytes(), "image/webp", "public, max-age=31536000, immutable")
|
|
m = re.fullmatch(r"/v1/jobs/([0-9a-f]{32})", path)
|
|
if m:
|
|
return self.send(200, {"id": m.group(1), **(pipeline.job_status(m.group(1)) or {"status": "missing"})})
|
|
m = re.fullmatch(r"/v1/illustrations/([0-9a-f]{32})(\.webp)?", path)
|
|
if not m:
|
|
return self.send(404, {"error": "not found"})
|
|
if m.group(2):
|
|
file = pipeline.image_path(m.group(1))
|
|
if not file.exists():
|
|
return self.send(404, {"error": "not ready"})
|
|
file = thumbnail(file, params.get("w"))
|
|
# Status links carry ?v=<mtime>, so a replaced image gets a new URL.
|
|
return self.send(200, file.read_bytes(), "image/webp", "public, max-age=31536000, immutable")
|
|
return self.send(200, pipeline.illustration_status(m.group(1)))
|
|
|
|
def proxy(self, path, query="", body=None):
|
|
"""Forward to the fetcher; searches ask every source and take about a minute."""
|
|
req = urllib.request.Request(f"{FETCHER}{path}" + (f"?{query}" if query else ""), body,
|
|
{"Content-Type": "application/json"} if body else {})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=240) as res:
|
|
return self.send(res.status, res.read())
|
|
except urllib.error.HTTPError as e:
|
|
return self.send(e.code, e.read())
|
|
except OSError:
|
|
return self.send(503, {"error": "the fetcher is not running"})
|
|
|
|
def do_DELETE(self):
|
|
m = re.fullmatch(r"/v1/illustrations/([0-9a-f]{32})", self.path.split("?", 1)[0])
|
|
if not m:
|
|
return self.send(404, {"error": "not found"})
|
|
job_id = m.group(1)
|
|
if not pipeline.cancel(job_id):
|
|
return self.send(409, {"error": "already painting; it cannot be stopped midway"})
|
|
if not pipeline.image_path(job_id).exists():
|
|
store.set_skipped(job_id, True)
|
|
self.send(200, pipeline.illustration_status(job_id))
|
|
|
|
def do_POST(self):
|
|
path = self.path.split("?", 1)[0]
|
|
if path == "/v1/library/fetch":
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
if not 0 < length <= 10_000:
|
|
return self.send(413, {"error": "body must be 1..10000 bytes"})
|
|
return self.proxy("/fetch", body=self.rfile.read(length))
|
|
handlers = {"/v1/illustrations": self.post_illustration, "/v1/settings": self.post_settings,
|
|
"/v1/styles": self.post_styles,
|
|
"/v1/progress": self.post_progress, "/v1/characters": self.post_character}
|
|
if path not in handlers:
|
|
return self.send(404, {"error": "not found"})
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
limit = MAX_UPLOAD * 2 if path == "/v1/characters" else MAX_BODY
|
|
if length <= 0 or length > limit:
|
|
return self.send(413, {"error": f"body must be 1..{limit} bytes"})
|
|
try:
|
|
req = json.loads(self.rfile.read(length))
|
|
if not isinstance(req, dict):
|
|
raise ValueError("body must be a JSON object")
|
|
handlers[path](req)
|
|
except (ValueError, json.JSONDecodeError) as e:
|
|
return self.send(400, {"error": str(e)})
|
|
|
|
def post_settings(self, req):
|
|
self.send(200, store.put_settings(text_field(req, "novel", required=True), req.get("every"),
|
|
req.get("paused", False), req.get("mode", "every")))
|
|
|
|
def post_illustration(self, req):
|
|
regenerate = req.get("regenerate") is True
|
|
slot = req.get("slot", 1)
|
|
if slot not in (1, 2):
|
|
raise ValueError("'slot' must be 1 or 2")
|
|
req = parse_illustration(req)
|
|
job_id = pipeline.slot_id(req["novel"], req["chapter"], slot)
|
|
store.set_skipped(job_id, False) # an explicit request overrides an earlier skip
|
|
if regenerate:
|
|
if pipeline.job_status(job_id) is None:
|
|
for suffix in (".webp", ".json"):
|
|
(IMAGES / f"{job_id}{suffix}").unlink(missing_ok=True)
|
|
store.forget_illustration(job_id)
|
|
if not pipeline.image_path(job_id).exists():
|
|
pipeline.submit(job_id, "illustration", pipeline.USER, pipeline.generate, req, slot)
|
|
else:
|
|
# Auto mode may want a second image for the chapters rated most visual.
|
|
pipeline.request_chapter(req, pipeline.USER, max(1, store.images_due(req["novel"], req["position"])))
|
|
current = pipeline.chapter_status(req["novel"], req["chapter"])
|
|
self.send(202 if current["status"] != "done" else 200, current)
|
|
|
|
def post_progress(self, req):
|
|
novel = text_field(req, "novel", required=True)
|
|
source = text_field(req, "source")
|
|
chapter = text_field(req, "chapter")
|
|
position = position_field(req)
|
|
gap, last = store.record_progress(novel, position)
|
|
recap_due = bool(gap and gap > RECAP_AFTER and position and store.summaries_before(novel, position, 1))
|
|
# Kavita calls can be slow or down; never hold up the reader for them.
|
|
threading.Thread(target=indexer.register, args=(novel, source, chapter), daemon=True).start()
|
|
if position is not None:
|
|
pipeline.drop_behind(novel, position)
|
|
prefetching = pipeline.prefetch_next(novel, source, chapter, position, text_field(req, "title"))
|
|
# Auto decisions live here (they need pre-reading's scores for "key" mode).
|
|
self.send(200, {"recap_due": recap_due, "prefetching": prefetching,
|
|
"due": store.images_due(novel, position),
|
|
"next_due": store.next_due(novel, position) if position is not None else None})
|
|
|
|
def post_styles(self, req):
|
|
novel = text_field(req, "novel", required=True)
|
|
if req.get("action") == "propose":
|
|
job_id = key(novel, "styles")
|
|
pipeline.submit(job_id, "styles", pipeline.USER, pipeline.propose_styles_job, novel)
|
|
return self.send(202, {"id": job_id, **(pipeline.job_status(job_id) or {"status": "queued"})})
|
|
if req.get("action") == "choose":
|
|
index = req.get("index")
|
|
if type(index) is not int:
|
|
raise ValueError("'index' must be an integer")
|
|
return self.send(200, pipeline.choose_style(novel, index))
|
|
raise ValueError("'action' must be propose or choose")
|
|
|
|
def post_character(self, req):
|
|
"""Character tools from the reader's cards and the wiki's Edit sheet."""
|
|
novel = text_field(req, "novel", required=True)
|
|
name = text_field(req, "name", required=True, limit=80)
|
|
action = req.get("action")
|
|
try:
|
|
if action == "merge":
|
|
into = text_field(req, "into", required=True, limit=80)
|
|
if not store.merge_characters(novel, into, name):
|
|
return self.send(404, {"error": "unknown characters"})
|
|
return self.send(200, {"merged": True})
|
|
if action == "edit":
|
|
aliases = req.get("aliases")
|
|
if aliases is not None and (not isinstance(aliases, list) or len(aliases) > 40
|
|
or not all(isinstance(a, str) and 0 < len(a) <= 80 for a in aliases)):
|
|
raise ValueError("'aliases' must be a list of up to 40 short strings")
|
|
role = req.get("role")
|
|
if role is not None and (not isinstance(role, str) or len(role) > 400):
|
|
raise ValueError("'role' must be a string of at most 400 characters")
|
|
new = store.edit_character(novel, name, text_field(req, "new_name", limit=80).strip() or None,
|
|
role, [a.strip() for a in aliases] if aliases is not None else None)
|
|
return self.send(200, {"name": new})
|
|
if action == "delete":
|
|
store.delete_character(novel, name)
|
|
return self.send(200, {"deleted": True})
|
|
if action == "split":
|
|
new_name = text_field(req, "new_name", required=True, limit=80).strip()
|
|
labels, aliases = req.get("looks"), req.get("aliases") or []
|
|
if not isinstance(labels, list) or not isinstance(aliases, list):
|
|
raise ValueError("'looks' and 'aliases' must be lists")
|
|
store.split_character(novel, name, new_name, [a for a in aliases if isinstance(a, str)],
|
|
[l for l in labels if isinstance(l, str)], text_field(req, "role", limit=400))
|
|
return self.send(200, {"name": new_name})
|
|
|
|
# Actions on one look of the character.
|
|
label = text_field(req, "label", required=True, limit=60)
|
|
with store.connect() as conn:
|
|
if label not in {l["label"] for l in store.looks_of(conn, novel, name)}:
|
|
return self.send(404, {"error": "unknown character or look"})
|
|
if action == "redraw":
|
|
hint = text_field(req, "hint", limit=MAX_HINT).strip()
|
|
job_id = key(novel, name, label, "portrait")
|
|
pipeline.submit(job_id, "portrait", pipeline.USER, pipeline.redraw_job, novel, name, label, hint)
|
|
return self.send(202, {"id": job_id, **(pipeline.job_status(job_id) or {"status": "queued"})})
|
|
if action == "upload":
|
|
image = pipeline.save_upload(novel, name, label, text_field(req, "image", required=True,
|
|
limit=MAX_UPLOAD * 2))
|
|
job_id = key(novel, name, label, "describe")
|
|
pipeline.run_now(job_id, "portrait", pipeline.describe_job, novel, name, label, image)
|
|
return self.send(202, {"id": job_id, **(pipeline.job_status(job_id) or {"status": "queued"})})
|
|
if action == "look":
|
|
position = position_field(req)
|
|
if position is None:
|
|
raise ValueError("'position' is required")
|
|
with store.connect() as conn:
|
|
store.set_timeline(conn, novel, name, position, label)
|
|
card = next((c for c in store.codex(novel, position)["characters"] if c["name"] == name), None)
|
|
return self.send(200, card or {})
|
|
if action == "edit_look":
|
|
since = req.get("since")
|
|
if since is not None and (type(since) is not int or since < 0):
|
|
raise ValueError("'since' must be a chapter position")
|
|
description = req.get("description")
|
|
if description is not None and (not isinstance(description, str) or len(description) > 600):
|
|
raise ValueError("'description' must be a string of at most 600 characters")
|
|
new = store.edit_look(novel, name, label, text_field(req, "new_label", limit=60).strip() or None,
|
|
description, since)
|
|
return self.send(200, {"label": new})
|
|
if action == "delete_look":
|
|
store.delete_look(novel, name, label, text_field(req, "into", required=True, limit=60))
|
|
return self.send(200, {"deleted": True})
|
|
except LookupError as e:
|
|
return self.send(404, {"error": str(e)})
|
|
raise ValueError("unknown action")
|
|
|
|
def log_message(self, fmt, *args):
|
|
print(f"{self.address_string()} {fmt % args}", file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
store.migrate()
|
|
pipeline.resume()
|
|
threading.Thread(target=pipeline.worker, daemon=True).start()
|
|
threading.Thread(target=indexer.run, daemon=True).start()
|
|
ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever()
|