From 1debde767375a8cda4487ac20b7a17e770d2b557 Mon Sep 17 00:00:00 2001 From: Hanashi Date: Thu, 6 Aug 2026 02:45:34 -0400 Subject: [PATCH] =?UTF-8?q?bench:=20v2=20reference=20set=20=E2=80=94=20JP?= =?UTF-8?q?=20stills,=20Ashitaka/Bebop,=20forests,=20volcanos=20(up=20to?= =?UTF-8?q?=2023k=20px)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- bench/minimax_bench.py | 21 ++++--- scripts/pod/fetch-bench-images-v2.py | 92 ++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 9 deletions(-) create mode 100644 scripts/pod/fetch-bench-images-v2.py diff --git a/bench/minimax_bench.py b/bench/minimax_bench.py index 44a8bbd..e6436e4 100644 --- a/bench/minimax_bench.py +++ b/bench/minimax_bench.py @@ -41,12 +41,14 @@ VIDEO_VAE = "minimax_h3_video_vae_fp16.safetensors" AUDIO_VAE = "minimax_h3_audio_vae_fp32.safetensors" FL2VA_DEFAULT = "minimax_h3_fl2va_pruned_int8_convrot.safetensors" REF2VA_DEFAULT = "minimax_h3_ref2va_pruned_int8_convrot.safetensors" -# Real images fetched by scripts/pod/fetch-bench-images.py (anime characters, -# film stills, giant panoramas) — realistic detail load for the ref encoder. -BENCH_IMAGE = "bench_movie_01.jpg" -BENCH_REF_SET = ["bench_anime_01.jpg", "bench_anime_02.jpg", "bench_anime_03.jpg", - "bench_movie_01.jpg", "bench_movie_02.jpg", "bench_movie_03.jpg", - "bench_big_01.jpg", "bench_big_02.jpg"] +# Real images fetched by scripts/pod/fetch-bench-images-v2.py: Jurassic Park +# stills, anime characters (Ashitaka, Spike), forests, volcanos — large files +# (up to 23040x3840) for realistic reference-encoder load. +BENCH_IMAGE = "bench2_jp_01.jpg" +BENCH_REF_SET = ["bench2_jp_01.jpg", "bench2_jp_gate_01.png", + "bench2_ashitaka_01.jpg", "bench2_bebop_01.jpg", + "bench2_forest_01.jpg", "bench2_forest_02.jpg", + "bench2_volcano_01.jpg", "bench2_volcano_02.png"] T2V_PROMPT = ( "Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, " @@ -68,9 +70,10 @@ def ref2v_prompt(ref_count): return REF2V_PROMPT tags = ", ".join(f"" for i in range(1, ref_count + 1)) return ( - f"A sweeping cinematic montage that travels through the {ref_count} city scenes " - f"from {tags} in order, crossfading between their times of day and color moods, " - "camera drifting forward the whole time, ambient city sounds evolving with each scene." + f"A sweeping cinematic montage inspired by {tags}: dinosaurs stalking through " + "ancient forests, an anime hero surveying a volcanic ridge, crossfading between " + "the scenes and moods of each reference in order, camera drifting forward the " + "whole time, an adventurous orchestral score building throughout." ) diff --git a/scripts/pod/fetch-bench-images-v2.py b/scripts/pod/fetch-bench-images-v2.py new file mode 100644 index 0000000..8389507 --- /dev/null +++ b/scripts/pod/fetch-bench-images-v2.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Fetch the v2 real-reference benchmark set into /ComfyUI/input. + +Large (min-width-enforced) images: Jurassic Park stills, anime characters +(Ashitaka, Cowboy Bebop), forests, volcanos. Sources are MediaWiki APIs +(Fandom wikis + Wikimedia Commons). Internal test assets only — not +redistributed with the repo. + +Produces bench2_* files + bench2_manifest.json. Re-runnable; skips existing. +""" +import json +import urllib.parse +import urllib.request +from pathlib import Path + +OUT = Path("/ComfyUI/input") +UA = {"User-Agent": "storyteller-bench/1.0 (internal model benchmarking)"} + +# (wiki api base, prefix, count, min_width, query) +SETS = [ + ("https://jurassicpark.fandom.com/api.php", "bench2_jp", 2, 1400, "Tyrannosaurus rex"), + ("https://jurassicpark.fandom.com/api.php", "bench2_jp_gate", 1, 1200, "Jurassic Park gate"), + ("https://ghibli.fandom.com/api.php", "bench2_ashitaka", 1, 900, "Ashitaka"), + ("https://cowboybebop.fandom.com/api.php", "bench2_bebop", 1, 900, "Spike Spiegel"), + ("https://commons.wikimedia.org/w/api.php", "bench2_forest", 2, 4000, "old growth forest filetype:bitmap"), + ("https://commons.wikimedia.org/w/api.php", "bench2_volcano", 2, 4000, "volcano eruption lava filetype:bitmap"), +] + + +def api(base, params): + q = urllib.parse.urlencode({**params, "format": "json"}) + req = urllib.request.Request(f"{base}?{q}", headers=UA) + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read()) + + +def search_images(base, query, need, min_width): + data = api(base, { + "action": "query", "generator": "search", + "gsrsearch": query, "gsrnamespace": 6, "gsrlimit": 50, + "prop": "imageinfo", "iiprop": "url|size|mime", + }) + pages = (data.get("query") or {}).get("pages", {}) + hits = [] + for p in sorted(pages.values(), key=lambda p: p.get("index", 99)): + ii = (p.get("imageinfo") or [{}])[0] + if ii.get("mime") in ("image/jpeg", "image/png") and ii.get("width", 0) >= min_width: + hits.append((ii["url"], ii["width"], ii["height"])) + if len(hits) >= need: + break + return hits + + +def fetch(url, dest): + req = urllib.request.Request(url, headers=UA) + with urllib.request.urlopen(req, timeout=180) as r, open(dest, "wb") as f: + f.write(r.read()) + + +def main(): + OUT.mkdir(parents=True, exist_ok=True) + manifest = {} + for base, prefix, count, min_width, query in SETS: + try: + hits = search_images(base, query, count, min_width) + except Exception as e: + print(f"WARNING: search failed for {prefix}: {e}") + hits = [] + if len(hits) < count: + # fall back: halve the width requirement once + try: + hits = search_images(base, query, count, min_width // 2) + except Exception: + pass + if len(hits) < count: + print(f"WARNING: only {len(hits)}/{count} for {prefix} ({query})") + for i, (url, w, h) in enumerate(hits, start=1): + ext = ".png" if ".png" in url.lower() else ".jpg" + name = f"{prefix}_{i:02d}{ext}" + dest = OUT / name + if dest.exists(): + print(f"skip {name}") + else: + print(f"fetch {name} <- {w}x{h}") + fetch(url, dest) + manifest[name] = {"source": url, "width": w, "height": h} + (OUT / "bench2_manifest.json").write_text(json.dumps(manifest, indent=1)) + print(f"done: {len(manifest)} images") + + +if __name__ == "__main__": + main()