storytold/cloud-worker
2
0
Fork 0
mirror of https://github.com/storytold/cloud-worker.git synced 2026-10-09 00:09:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

bench: multi-reference ref2v support (1-8 ref images, match/max sizing)

Browse Source 
- refheavy suite: ref2v saturation with 1/4/8 refs, the priority modality
- family suite: reduced grid for int8/bf16 family comparisons
- 8 distinct 1344x768 reference scenes generated into /ComfyUI/input

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Hanashi
2026-08-06 00:45:49 -04:00
parent 23a755e394
commit 52f8a79f35
2 changed files with 104 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bench/minimax_bench.py
+76 -11
View File
@@ -42,6 +42,7 @@ AUDIO_VAE = "minimax_h3_audio_vae_fp32.safetensors"
FL2VA_DEFAULT = "minimax_h3_fl2va_pruned_int8_convrot.safetensors"
REF2VA_DEFAULT = "minimax_h3_ref2va_pruned_int8_convrot.safetensors"
BENCH_IMAGE = "bench_ref_1344x768.png" # created by scripts/pod/make-test-images.py
BENCH_REF_SET = [f"bench_ref_{i:02d}.png" for i in range(1, 9)] # multi-ref pool
T2V_PROMPT = (
"Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, "
@@ -58,6 +59,17 @@ REF2V_PROMPT = (
)
def ref2v_prompt(ref_count):
if ref_count <= 1:
return REF2V_PROMPT
tags = ", ".join(f"<Picture {i}>" for i in range(1, ref_count + 1))
return (
f"A sweeping cinematic montage that travels through the {ref_count} city scenes "
f"from {tags} in order, crossfading between their times of day and color moods, "
"camera drifting forward the whole time, ambient city sounds evolving with each scene."
)
def snap_length(seconds: float) -> int:
"""Duration in seconds -> valid H3 frame count (24 fps, n % 17 == 5, snapped up)."""
n = max(5, round(seconds * 24))
@@ -66,7 +78,7 @@ def snap_length(seconds: float) -> int:
def build_graph(task, model_file, prompt, width, height, length, steps, seed,
sampler="res_multistep", scheduler="simple", image=BENCH_IMAGE,
ref_image_size="match", filename_prefix="bench/run"):
ref_image_size="match", ref_count=1, filename_prefix="bench/run"):
g = {
"1": {"class_type": "UNETLoader",
"inputs": {"unet_name": model_file, "weight_dtype": "default"}},
@@ -102,11 +114,20 @@ def build_graph(task, model_file, prompt, width, height, length, steps, seed,
"inputs": {"clip": ["2", 0], "vae": ["3", 0], "first_frame": ["10", 0],
**common}}
elif task == "ref2v":
g["10"] = {"class_type": "LoadImage", "inputs": {"image": image}}
ref_inputs = {}
if ref_count <= 1:
g["10"] = {"class_type": "LoadImage", "inputs": {"image": image}}
ref_inputs["ref_images.ref_image_0"] = ["10", 0]
else:
for i in range(ref_count):
nid = str(20 + i)
g[nid] = {"class_type": "LoadImage",
"inputs": {"image": BENCH_REF_SET[i % len(BENCH_REF_SET)]}}
ref_inputs[f"ref_images.ref_image_{i}"] = [nid, 0]
g["5"] = {"class_type": "MiniMaxH3ReferenceToVideo",
"inputs": {"clip": ["2", 0], "vae": ["3", 0], "audio_vae": ["4", 0],
"ref_image_size": ref_image_size,
"ref_images.ref_image_0": ["10", 0], **common}}
**ref_inputs, **common}}
else:
raise ValueError(f"unknown task {task}")
return g
@@ -144,7 +165,8 @@ def run_one(api, cfg, poll=2.0, timeout=3600):
"""Submit one benchmark run; returns result dict."""
graph = build_graph(**{k: v for k, v in cfg.items() if k in (
"task", "model_file", "prompt", "width", "height", "length", "steps", "seed",
"sampler", "scheduler", "image", "ref_image_size", "filename_prefix")})
"sampler", "scheduler", "image", "ref_image_size", "ref_count",
"filename_prefix")})
peak = {"vram": 0}
stop = threading.Event()
@@ -205,8 +227,8 @@ def append_results(row, outdir):
f.write(json.dumps({**row, "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}) + "\n")
csv_path = outdir / "results.csv"
fields = ["ts", "label", "task", "model_file", "width", "height", "seconds", "length",
"steps", "sampler", "scheduler", "ref_image_size", "warm", "ok", "wall_s",
"exec_s", "peak_vram_gb", "outputs", "error"]
"steps", "sampler", "scheduler", "ref_image_size", "ref_count", "warm",
"ok", "wall_s", "exec_s", "peak_vram_gb", "outputs", "error"]
new = not csv_path.exists()
with csv_path.open("a", newline="") as f:
w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
@@ -221,13 +243,16 @@ def default_prompt(task):
def make_cfg(task, model_file, width, height, seconds, steps, seed, label, warm,
sampler="res_multistep", scheduler="simple", ref_image_size="match"):
sampler="res_multistep", scheduler="simple", ref_image_size="match",
ref_count=1):
prompt = ref2v_prompt(ref_count) if task == "ref2v" else default_prompt(task)
return {
"label": label, "task": task, "model_file": model_file,
"prompt": default_prompt(task), "width": width, "height": height,
"prompt": prompt, "width": width, "height": height,
"seconds": seconds, "length": snap_length(seconds), "steps": steps,
"seed": seed, "sampler": sampler, "scheduler": scheduler,
"image": BENCH_IMAGE, "ref_image_size": ref_image_size, "warm": warm,
"image": BENCH_IMAGE, "ref_image_size": ref_image_size,
"ref_count": ref_count, "warm": warm,
"filename_prefix": f"bench/{label}",
}
@@ -241,6 +266,39 @@ def suite_quick(fl2va, ref2va, prefix=""):
]
def suite_refheavy(ref2va, prefix=""):
"""Ref2v saturation: 1/4/8 reference images, match vs max sizing.
Ref tokens ride through every sampling step, so this measures how reference
context load scales cost. 864x480 x 5 s base config, plus one max-res run.
"""
cfgs = []
for n in (1, 4, 8):
cfgs.append(make_cfg("ref2v", ref2va, 864, 480, 5, 20, 3,
f"{prefix}refheavy_{n}ref_match", warm=True, ref_count=n))
for n in (4, 8):
cfgs.append(make_cfg("ref2v", ref2va, 864, 480, 5, 20, 3,
f"{prefix}refheavy_{n}ref_max", warm=True,
ref_image_size="max", ref_count=n))
cfgs.append(make_cfg("ref2v", ref2va, 1344, 768, 5, 20, 3,
f"{prefix}refheavy_8ref_match_1344", warm=True, ref_count=8))
return cfgs
def suite_family(fl2va, ref2va, prefix=""):
"""Reduced grid for comparing weight families without the full sweep:
key t2v/i2v/ref2v points at default + max res, short + long duration."""
return [
make_cfg("t2v", fl2va, 864, 480, 5, 20, 2, f"{prefix}fam_t2v_864x480_5s", warm=False),
make_cfg("t2v", fl2va, 864, 480, 15, 20, 2, f"{prefix}fam_t2v_864x480_15s", warm=True),
make_cfg("t2v", fl2va, 1344, 768, 5, 20, 2, f"{prefix}fam_t2v_1344x768_5s", warm=True),
make_cfg("i2v", fl2va, 864, 480, 5, 20, 2, f"{prefix}fam_i2v_864x480_5s", warm=True),
make_cfg("ref2v", ref2va, 864, 480, 5, 20, 2, f"{prefix}fam_ref2v_864x480_5s", warm=False),
make_cfg("ref2v", ref2va, 864, 480, 15, 20, 2, f"{prefix}fam_ref2v_864x480_15s", warm=True),
make_cfg("ref2v", ref2va, 1344, 768, 5, 20, 2, f"{prefix}fam_ref2v_1344x768_5s", warm=True),
]
def suite_sweep(fl2va, ref2va, prefix=""):
"""Full sweep: tasks x durations x resolutions at fixed 20 steps.
@@ -278,7 +336,7 @@ def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--host", default=DEFAULT_HOST)
ap.add_argument("--suite", choices=["quick", "sweep"])
ap.add_argument("--suite", choices=["quick", "sweep", "refheavy", "family"])
ap.add_argument("--task", choices=["t2v", "i2v", "ref2v"])
ap.add_argument("--model", action="append", default=[],
help="override model files, e.g. fl2va=file.safetensors (repeatable)")
@@ -290,6 +348,8 @@ def main():
ap.add_argument("--sampler", default="res_multistep")
ap.add_argument("--scheduler", default="simple")
ap.add_argument("--ref-image-size", default="match", choices=["match", "max"])
ap.add_argument("--ref-count", type=int, default=1,
help="number of reference images for ref2v (1-8)")
ap.add_argument("--label", default=None)
ap.add_argument("--prefix", default="", help="label prefix for suites, e.g. prunedint8_")
ap.add_argument("--warm", action="store_true", help="mark run as warm (model preloaded)")
@@ -306,12 +366,17 @@ def main():
cfgs = suite_quick(models["fl2va"], models["ref2va"], args.prefix)
elif args.suite == "sweep":
cfgs = suite_sweep(models["fl2va"], models["ref2va"], args.prefix)
elif args.suite == "refheavy":
cfgs = suite_refheavy(models["ref2va"], args.prefix)
elif args.suite == "family":
cfgs = suite_family(models["fl2va"], models["ref2va"], args.prefix)
elif args.task:
model = models["ref2va"] if args.task == "ref2v" else models["fl2va"]
label = args.label or f"{args.task}_{args.width}x{args.height}_{args.seconds}s"
cfgs = [make_cfg(args.task, model, args.width, args.height, args.seconds,
args.steps, args.seed, label, args.warm,
args.sampler, args.scheduler, args.ref_image_size)]
args.sampler, args.scheduler, args.ref_image_size,
args.ref_count)]
else:
ap.error("need --suite or --task")