bench: v2 reference set — JP stills, Ashitaka/Bebop, forests, volcanos (up to 23k px)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

bench/minimax_bench.py

@@ -41,12 +41,14 @@ VIDEO_VAE = "minimax_h3_video_vae_fp16.safetensors"
 AUDIO_VAE = "minimax_h3_audio_vae_fp32.safetensors"
 FL2VA_DEFAULT = "minimax_h3_fl2va_pruned_int8_convrot.safetensors"
 REF2VA_DEFAULT = "minimax_h3_ref2va_pruned_int8_convrot.safetensors"
-# Real images fetched by scripts/pod/fetch-bench-images.py (anime characters,
-# film stills, giant panoramas) — realistic detail load for the ref encoder.
-BENCH_IMAGE = "bench_movie_01.jpg"
-BENCH_REF_SET = ["bench_anime_01.jpg", "bench_anime_02.jpg", "bench_anime_03.jpg",
-                 "bench_movie_01.jpg", "bench_movie_02.jpg", "bench_movie_03.jpg",
-                 "bench_big_01.jpg", "bench_big_02.jpg"]
+# Real images fetched by scripts/pod/fetch-bench-images-v2.py: Jurassic Park
+# stills, anime characters (Ashitaka, Spike), forests, volcanos — large files
+# (up to 23040x3840) for realistic reference-encoder load.
+BENCH_IMAGE = "bench2_jp_01.jpg"
+BENCH_REF_SET = ["bench2_jp_01.jpg", "bench2_jp_gate_01.png",
+                 "bench2_ashitaka_01.jpg", "bench2_bebop_01.jpg",
+                 "bench2_forest_01.jpg", "bench2_forest_02.jpg",
+                 "bench2_volcano_01.jpg", "bench2_volcano_02.png"]
 
 T2V_PROMPT = (
     "Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, "
@@ -68,9 +70,10 @@ def ref2v_prompt(ref_count):
         return REF2V_PROMPT
     tags = ", ".join(f"<Picture {i}>" for i in range(1, ref_count + 1))
     return (
-        f"A sweeping cinematic montage that travels through the {ref_count} city scenes "
-        f"from {tags} in order, crossfading between their times of day and color moods, "
-        "camera drifting forward the whole time, ambient city sounds evolving with each scene."
+        f"A sweeping cinematic montage inspired by {tags}: dinosaurs stalking through "
+        "ancient forests, an anime hero surveying a volcanic ridge, crossfading between "
+        "the scenes and moods of each reference in order, camera drifting forward the "
+        "whole time, an adventurous orchestral score building throughout."
     )
 
 

scripts/pod/fetch-bench-images-v2.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Fetch the v2 real-reference benchmark set into /ComfyUI/input.
+
+Large (min-width-enforced) images: Jurassic Park stills, anime characters
+(Ashitaka, Cowboy Bebop), forests, volcanos. Sources are MediaWiki APIs
+(Fandom wikis + Wikimedia Commons). Internal test assets only — not
+redistributed with the repo.
+
+Produces bench2_* files + bench2_manifest.json. Re-runnable; skips existing.
+"""
+import json
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+OUT = Path("/ComfyUI/input")
+UA = {"User-Agent": "storyteller-bench/1.0 (internal model benchmarking)"}
+
+# (wiki api base, prefix, count, min_width, query)
+SETS = [
+    ("https://jurassicpark.fandom.com/api.php", "bench2_jp",       2, 1400, "Tyrannosaurus rex"),
+    ("https://jurassicpark.fandom.com/api.php", "bench2_jp_gate",  1, 1200, "Jurassic Park gate"),
+    ("https://ghibli.fandom.com/api.php",       "bench2_ashitaka", 1, 900,  "Ashitaka"),
+    ("https://cowboybebop.fandom.com/api.php",  "bench2_bebop",    1, 900,  "Spike Spiegel"),
+    ("https://commons.wikimedia.org/w/api.php", "bench2_forest",   2, 4000, "old growth forest filetype:bitmap"),
+    ("https://commons.wikimedia.org/w/api.php", "bench2_volcano",  2, 4000, "volcano eruption lava filetype:bitmap"),
+]
+
+
+def api(base, params):
+    q = urllib.parse.urlencode({**params, "format": "json"})
+    req = urllib.request.Request(f"{base}?{q}", headers=UA)
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return json.loads(r.read())
+
+
+def search_images(base, query, need, min_width):
+    data = api(base, {
+        "action": "query", "generator": "search",
+        "gsrsearch": query, "gsrnamespace": 6, "gsrlimit": 50,
+        "prop": "imageinfo", "iiprop": "url|size|mime",
+    })
+    pages = (data.get("query") or {}).get("pages", {})
+    hits = []
+    for p in sorted(pages.values(), key=lambda p: p.get("index", 99)):
+        ii = (p.get("imageinfo") or [{}])[0]
+        if ii.get("mime") in ("image/jpeg", "image/png") and ii.get("width", 0) >= min_width:
+            hits.append((ii["url"], ii["width"], ii["height"]))
+        if len(hits) >= need:
+            break
+    return hits
+
+
+def fetch(url, dest):
+    req = urllib.request.Request(url, headers=UA)
+    with urllib.request.urlopen(req, timeout=180) as r, open(dest, "wb") as f:
+        f.write(r.read())
+
+
+def main():
+    OUT.mkdir(parents=True, exist_ok=True)
+    manifest = {}
+    for base, prefix, count, min_width, query in SETS:
+        try:
+            hits = search_images(base, query, count, min_width)
+        except Exception as e:
+            print(f"WARNING: search failed for {prefix}: {e}")
+            hits = []
+        if len(hits) < count:
+            # fall back: halve the width requirement once
+            try:
+                hits = search_images(base, query, count, min_width // 2)
+            except Exception:
+                pass
+        if len(hits) < count:
+            print(f"WARNING: only {len(hits)}/{count} for {prefix} ({query})")
+        for i, (url, w, h) in enumerate(hits, start=1):
+            ext = ".png" if ".png" in url.lower() else ".jpg"
+            name = f"{prefix}_{i:02d}{ext}"
+            dest = OUT / name
+            if dest.exists():
+                print(f"skip {name}")
+            else:
+                print(f"fetch {name} <- {w}x{h}")
+                fetch(url, dest)
+            manifest[name] = {"source": url, "width": w, "height": h}
+    (OUT / "bench2_manifest.json").write_text(json.dumps(manifest, indent=1))
+    print(f"done: {len(manifest)} images")
+
+
+if __name__ == "__main__":
+    main()