diff --git a/bench/minimax_bench.py b/bench/minimax_bench.py index 5d1a2b6..8a6f26a 100644 --- a/bench/minimax_bench.py +++ b/bench/minimax_bench.py @@ -42,6 +42,7 @@ AUDIO_VAE = "minimax_h3_audio_vae_fp32.safetensors" FL2VA_DEFAULT = "minimax_h3_fl2va_pruned_int8_convrot.safetensors" REF2VA_DEFAULT = "minimax_h3_ref2va_pruned_int8_convrot.safetensors" BENCH_IMAGE = "bench_ref_1344x768.png" # created by scripts/pod/make-test-images.py +BENCH_REF_SET = [f"bench_ref_{i:02d}.png" for i in range(1, 9)] # multi-ref pool T2V_PROMPT = ( "Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, " @@ -58,6 +59,17 @@ REF2V_PROMPT = ( ) +def ref2v_prompt(ref_count): + if ref_count <= 1: + return REF2V_PROMPT + tags = ", ".join(f"" for i in range(1, ref_count + 1)) + return ( + f"A sweeping cinematic montage that travels through the {ref_count} city scenes " + f"from {tags} in order, crossfading between their times of day and color moods, " + "camera drifting forward the whole time, ambient city sounds evolving with each scene." + ) + + def snap_length(seconds: float) -> int: """Duration in seconds -> valid H3 frame count (24 fps, n % 17 == 5, snapped up).""" n = max(5, round(seconds * 24)) @@ -66,7 +78,7 @@ def snap_length(seconds: float) -> int: def build_graph(task, model_file, prompt, width, height, length, steps, seed, sampler="res_multistep", scheduler="simple", image=BENCH_IMAGE, - ref_image_size="match", filename_prefix="bench/run"): + ref_image_size="match", ref_count=1, filename_prefix="bench/run"): g = { "1": {"class_type": "UNETLoader", "inputs": {"unet_name": model_file, "weight_dtype": "default"}}, @@ -102,11 +114,20 @@ def build_graph(task, model_file, prompt, width, height, length, steps, seed, "inputs": {"clip": ["2", 0], "vae": ["3", 0], "first_frame": ["10", 0], **common}} elif task == "ref2v": - g["10"] = {"class_type": "LoadImage", "inputs": {"image": image}} + ref_inputs = {} + if ref_count <= 1: + g["10"] = {"class_type": "LoadImage", "inputs": {"image": image}} + ref_inputs["ref_images.ref_image_0"] = ["10", 0] + else: + for i in range(ref_count): + nid = str(20 + i) + g[nid] = {"class_type": "LoadImage", + "inputs": {"image": BENCH_REF_SET[i % len(BENCH_REF_SET)]}} + ref_inputs[f"ref_images.ref_image_{i}"] = [nid, 0] g["5"] = {"class_type": "MiniMaxH3ReferenceToVideo", "inputs": {"clip": ["2", 0], "vae": ["3", 0], "audio_vae": ["4", 0], "ref_image_size": ref_image_size, - "ref_images.ref_image_0": ["10", 0], **common}} + **ref_inputs, **common}} else: raise ValueError(f"unknown task {task}") return g @@ -144,7 +165,8 @@ def run_one(api, cfg, poll=2.0, timeout=3600): """Submit one benchmark run; returns result dict.""" graph = build_graph(**{k: v for k, v in cfg.items() if k in ( "task", "model_file", "prompt", "width", "height", "length", "steps", "seed", - "sampler", "scheduler", "image", "ref_image_size", "filename_prefix")}) + "sampler", "scheduler", "image", "ref_image_size", "ref_count", + "filename_prefix")}) peak = {"vram": 0} stop = threading.Event() @@ -205,8 +227,8 @@ def append_results(row, outdir): f.write(json.dumps({**row, "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}) + "\n") csv_path = outdir / "results.csv" fields = ["ts", "label", "task", "model_file", "width", "height", "seconds", "length", - "steps", "sampler", "scheduler", "ref_image_size", "warm", "ok", "wall_s", - "exec_s", "peak_vram_gb", "outputs", "error"] + "steps", "sampler", "scheduler", "ref_image_size", "ref_count", "warm", + "ok", "wall_s", "exec_s", "peak_vram_gb", "outputs", "error"] new = not csv_path.exists() with csv_path.open("a", newline="") as f: w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore") @@ -221,13 +243,16 @@ def default_prompt(task): def make_cfg(task, model_file, width, height, seconds, steps, seed, label, warm, - sampler="res_multistep", scheduler="simple", ref_image_size="match"): + sampler="res_multistep", scheduler="simple", ref_image_size="match", + ref_count=1): + prompt = ref2v_prompt(ref_count) if task == "ref2v" else default_prompt(task) return { "label": label, "task": task, "model_file": model_file, - "prompt": default_prompt(task), "width": width, "height": height, + "prompt": prompt, "width": width, "height": height, "seconds": seconds, "length": snap_length(seconds), "steps": steps, "seed": seed, "sampler": sampler, "scheduler": scheduler, - "image": BENCH_IMAGE, "ref_image_size": ref_image_size, "warm": warm, + "image": BENCH_IMAGE, "ref_image_size": ref_image_size, + "ref_count": ref_count, "warm": warm, "filename_prefix": f"bench/{label}", } @@ -241,6 +266,39 @@ def suite_quick(fl2va, ref2va, prefix=""): ] +def suite_refheavy(ref2va, prefix=""): + """Ref2v saturation: 1/4/8 reference images, match vs max sizing. + + Ref tokens ride through every sampling step, so this measures how reference + context load scales cost. 864x480 x 5 s base config, plus one max-res run. + """ + cfgs = [] + for n in (1, 4, 8): + cfgs.append(make_cfg("ref2v", ref2va, 864, 480, 5, 20, 3, + f"{prefix}refheavy_{n}ref_match", warm=True, ref_count=n)) + for n in (4, 8): + cfgs.append(make_cfg("ref2v", ref2va, 864, 480, 5, 20, 3, + f"{prefix}refheavy_{n}ref_max", warm=True, + ref_image_size="max", ref_count=n)) + cfgs.append(make_cfg("ref2v", ref2va, 1344, 768, 5, 20, 3, + f"{prefix}refheavy_8ref_match_1344", warm=True, ref_count=8)) + return cfgs + + +def suite_family(fl2va, ref2va, prefix=""): + """Reduced grid for comparing weight families without the full sweep: + key t2v/i2v/ref2v points at default + max res, short + long duration.""" + return [ + make_cfg("t2v", fl2va, 864, 480, 5, 20, 2, f"{prefix}fam_t2v_864x480_5s", warm=False), + make_cfg("t2v", fl2va, 864, 480, 15, 20, 2, f"{prefix}fam_t2v_864x480_15s", warm=True), + make_cfg("t2v", fl2va, 1344, 768, 5, 20, 2, f"{prefix}fam_t2v_1344x768_5s", warm=True), + make_cfg("i2v", fl2va, 864, 480, 5, 20, 2, f"{prefix}fam_i2v_864x480_5s", warm=True), + make_cfg("ref2v", ref2va, 864, 480, 5, 20, 2, f"{prefix}fam_ref2v_864x480_5s", warm=False), + make_cfg("ref2v", ref2va, 864, 480, 15, 20, 2, f"{prefix}fam_ref2v_864x480_15s", warm=True), + make_cfg("ref2v", ref2va, 1344, 768, 5, 20, 2, f"{prefix}fam_ref2v_1344x768_5s", warm=True), + ] + + def suite_sweep(fl2va, ref2va, prefix=""): """Full sweep: tasks x durations x resolutions at fixed 20 steps. @@ -278,7 +336,7 @@ def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--host", default=DEFAULT_HOST) - ap.add_argument("--suite", choices=["quick", "sweep"]) + ap.add_argument("--suite", choices=["quick", "sweep", "refheavy", "family"]) ap.add_argument("--task", choices=["t2v", "i2v", "ref2v"]) ap.add_argument("--model", action="append", default=[], help="override model files, e.g. fl2va=file.safetensors (repeatable)") @@ -290,6 +348,8 @@ def main(): ap.add_argument("--sampler", default="res_multistep") ap.add_argument("--scheduler", default="simple") ap.add_argument("--ref-image-size", default="match", choices=["match", "max"]) + ap.add_argument("--ref-count", type=int, default=1, + help="number of reference images for ref2v (1-8)") ap.add_argument("--label", default=None) ap.add_argument("--prefix", default="", help="label prefix for suites, e.g. prunedint8_") ap.add_argument("--warm", action="store_true", help="mark run as warm (model preloaded)") @@ -306,12 +366,17 @@ def main(): cfgs = suite_quick(models["fl2va"], models["ref2va"], args.prefix) elif args.suite == "sweep": cfgs = suite_sweep(models["fl2va"], models["ref2va"], args.prefix) + elif args.suite == "refheavy": + cfgs = suite_refheavy(models["ref2va"], args.prefix) + elif args.suite == "family": + cfgs = suite_family(models["fl2va"], models["ref2va"], args.prefix) elif args.task: model = models["ref2va"] if args.task == "ref2v" else models["fl2va"] label = args.label or f"{args.task}_{args.width}x{args.height}_{args.seconds}s" cfgs = [make_cfg(args.task, model, args.width, args.height, args.seconds, args.steps, args.seed, label, args.warm, - args.sampler, args.scheduler, args.ref_image_size)] + args.sampler, args.scheduler, args.ref_image_size, + args.ref_count)] else: ap.error("need --suite or --task") diff --git a/scripts/pod/make-test-images.py b/scripts/pod/make-test-images.py index 2aef228..1d7dd26 100644 --- a/scripts/pod/make-test-images.py +++ b/scripts/pod/make-test-images.py @@ -114,8 +114,35 @@ def bench_ref(): return img +# Distinct palettes for the multi-reference ref2v benchmark set. Each image is +# a different scene so 4-8 refs exercise genuinely different reference context. +REF_PALETTES = [ + ((200, 120, 60), (240, 200, 140)), # amber sunset + ((20, 30, 80), (90, 140, 200)), # blue dusk + ((10, 60, 50), (120, 200, 160)), # teal dawn + ((80, 20, 90), (200, 120, 220)), # violet night + ((160, 40, 30), (250, 150, 90)), # red dusk + ((30, 30, 30), (140, 140, 150)), # overcast grey + ((10, 20, 40), (40, 90, 160)), # deep night blue + ((190, 160, 40), (250, 240, 170)), # golden noon +] + + +def bench_ref_set(): + """8 distinct city scenes, bench_ref_01..08.png (1344x768).""" + for i, (top, bottom) in enumerate(REF_PALETTES, start=1): + img = city_backdrop(1344, 768, top, bottom, seed=100 + i) + d = ImageDraw.Draw(img) + # distinct celestial marker per scene so refs are visually distinct + x = 120 + i * 130 + d.ellipse([x, 80, x + 90, 170], + fill=(255, 240, 200) if i % 2 else (240, 240, 255)) + img.save(f"{OUT}/bench_ref_{i:02d}.png") + + if __name__ == "__main__": superboy().save(f"{OUT}/red_superboy_on_city_roof.png") mecha_dragon().save(f"{OUT}/mecha_dragon_lightning.png") bench_ref().save(f"{OUT}/bench_ref_1344x768.png") - print("wrote 3 images to", OUT) + bench_ref_set() + print("wrote 11 images to", OUT)