storytold/cloud-worker
2
0
Fork 0
mirror of https://github.com/storytold/cloud-worker.git synced 2026-10-17 00:23:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

reliability: failure-stats + output-validation tooling; persistent output dir

Browse Source 
- bench/failures.py: per-GPU success rates and error-class breakdown by config
- bench/validate_outputs.py: ffprobe every recorded output via /view
- finding: 24GB cards fail on multi-ref (TE vision OOM), not duration/size;
  100% of successful runs produce valid videos
- outputs now written to /workspace (survives pod resets)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Hanashi
2026-08-06 14:41:00 -04:00
parent fbc68bb749
commit a293d0a2c4
13 changed files with 308 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bench/4000/results.csv
+5
View File
@@ -1,3 +1,8 @@
ts,label,task,model_file,width,height,seconds,length,steps,sampler,scheduler,ref_image_size,ref_count,warm,ok,wall_s,exec_s,peak_vram_gb,outputs,error
2026-08-06T07:51:24Z,g4000_smoke_t2v,t2v,minimax_h3_fl2va_pruned_int8_convrot.safetensors,608,352,5.0,124,20,res_multistep,simple,match,1,False,False,135.0,,15.0,,server unreachable while polling: [Errno 104] Connection reset by peer
2026-08-06T07:58:16Z,g4000_smoke_t2v_b,t2v,minimax_h3_fl2va_pruned_int8_convrot.safetensors,608,352,5.0,124,20,res_multistep,simple,match,1,False,True,343.4,341.0,22.7,g4000_smoke_t2v_b_00001_.mp4,
2026-08-06T17:58:17Z,g4000_pi8_key_t2v_5s_r0,t2v,minimax_h3_fl2va_pruned_int8_convrot.safetensors,864,480,5,124,20,res_multistep,simple,match,1,False,True,559.7,557.0,20.4,g4000_pi8_key_t2v_5s_r0_00001_.mp4,
2026-08-06T18:07:31Z,g4000_pi8_key_t2v_5s_r1,t2v,minimax_h3_fl2va_pruned_int8_convrot.safetensors,864,480,5,124,20,res_multistep,simple,match,1,True,True,553.4,551.4,20.4,g4000_pi8_key_t2v_5s_r1_00001_.mp4,
2026-08-06T18:16:44Z,g4000_pi8_key_t2v_5s_r2,t2v,minimax_h3_fl2va_pruned_int8_convrot.safetensors,864,480,5,124,20,res_multistep,simple,match,1,True,True,553.4,550.6,20.4,g4000_pi8_key_t2v_5s_r2_00001_.mp4,
2026-08-06T18:26:34Z,g4000_pi8_key_ref2v_1ref_r0,ref2v,minimax_h3_ref2va_pruned_int8_convrot.safetensors,864,480,5,124,20,res_multistep,simple,match,1,False,True,589.8,587.8,19.0,g4000_pi8_key_ref2v_1ref_r0_00001_.mp4,
2026-08-06T18:36:16Z,g4000_pi8_key_ref2v_1ref_r1,ref2v,minimax_h3_ref2va_pruned_int8_convrot.safetensors,864,480,5,124,20,res_multistep,simple,match,1,True,True,582.4,580.1,19.0,g4000_pi8_key_ref2v_1ref_r1_00001_.mp4,
1 ts label task model_file width height seconds length steps sampler scheduler ref_image_size ref_count warm ok wall_s exec_s peak_vram_gb outputs error
2 2026-08-06T07:51:24Z g4000_smoke_t2v t2v minimax_h3_fl2va_pruned_int8_convrot.safetensors 608 352 5.0 124 20 res_multistep simple match 1 False False 135.0 15.0 server unreachable while polling: [Errno 104] Connection reset by peer
3 2026-08-06T07:58:16Z g4000_smoke_t2v_b t2v minimax_h3_fl2va_pruned_int8_convrot.safetensors 608 352 5.0 124 20 res_multistep simple match 1 False True 343.4 341.0 22.7 g4000_smoke_t2v_b_00001_.mp4
4 2026-08-06T17:58:17Z g4000_pi8_key_t2v_5s_r0 t2v minimax_h3_fl2va_pruned_int8_convrot.safetensors 864 480 5 124 20 res_multistep simple match 1 False True 559.7 557.0 20.4 g4000_pi8_key_t2v_5s_r0_00001_.mp4
5 2026-08-06T18:07:31Z g4000_pi8_key_t2v_5s_r1 t2v minimax_h3_fl2va_pruned_int8_convrot.safetensors 864 480 5 124 20 res_multistep simple match 1 True True 553.4 551.4 20.4 g4000_pi8_key_t2v_5s_r1_00001_.mp4
6 2026-08-06T18:16:44Z g4000_pi8_key_t2v_5s_r2 t2v minimax_h3_fl2va_pruned_int8_convrot.safetensors 864 480 5 124 20 res_multistep simple match 1 True True 553.4 550.6 20.4 g4000_pi8_key_t2v_5s_r2_00001_.mp4
7 2026-08-06T18:26:34Z g4000_pi8_key_ref2v_1ref_r0 ref2v minimax_h3_ref2va_pruned_int8_convrot.safetensors 864 480 5 124 20 res_multistep simple match 1 False True 589.8 587.8 19.0 g4000_pi8_key_ref2v_1ref_r0_00001_.mp4
8 2026-08-06T18:36:16Z g4000_pi8_key_ref2v_1ref_r1 ref2v minimax_h3_ref2va_pruned_int8_convrot.safetensors 864 480 5 124 20 res_multistep simple match 1 True True 582.4 580.1 19.0 g4000_pi8_key_ref2v_1ref_r1_00001_.mp4
bench/4000/results.jsonl
+5
View File
@@ -1,2 +1,7 @@
{"label": "g4000_smoke_t2v", "task": "t2v", "model_file": "minimax_h3_fl2va_pruned_int8_convrot.safetensors", "prompt": "Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, waves crashing on dark rocks below, seagulls circling, warm sunlight flares, sound of surf and wind, gentle orchestral swell.", "width": 608, "height": 352, "seconds": 5.0, "length": 124, "steps": 20, "seed": 7, "sampler": "res_multistep", "scheduler": "simple", "image": "bench2_jp_01.jpg", "ref_image_size": "match", "ref_count": 1, "warm": false, "filename_prefix": "bench/g4000_smoke_t2v", "ok": false, "status": "server_lost", "wall_s": 135.0, "exec_s": null, "peak_vram_gb": 15.0, "outputs": [], "error": "server unreachable while polling: [Errno 104] Connection reset by peer", "ts": "2026-08-06T07:51:24Z"}
{"label": "g4000_smoke_t2v_b", "task": "t2v", "model_file": "minimax_h3_fl2va_pruned_int8_convrot.safetensors", "prompt": "Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, waves crashing on dark rocks below, seagulls circling, warm sunlight flares, sound of surf and wind, gentle orchestral swell.", "width": 608, "height": 352, "seconds": 5.0, "length": 124, "steps": 20, "seed": 9, "sampler": "res_multistep", "scheduler": "simple", "image": "bench2_jp_01.jpg", "ref_image_size": "match", "ref_count": 1, "warm": false, "filename_prefix": "bench/g4000_smoke_t2v_b", "ok": true, "status": "success", "wall_s": 343.4, "exec_s": 341.0, "peak_vram_gb": 22.7, "outputs": ["g4000_smoke_t2v_b_00001_.mp4"], "error": null, "ts": "2026-08-06T07:58:16Z"}
{"label": "g4000_pi8_key_t2v_5s_r0", "task": "t2v", "model_file": "minimax_h3_fl2va_pruned_int8_convrot.safetensors", "prompt": "Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, waves crashing on dark rocks below, seagulls circling, warm sunlight flares, sound of surf and wind, gentle orchestral swell.", "width": 864, "height": 480, "seconds": 5, "length": 124, "steps": 20, "seed": 10, "sampler": "res_multistep", "scheduler": "simple", "image": "bench2_jp_01.jpg", "ref_image_size": "match", "ref_count": 1, "warm": false, "filename_prefix": "bench/g4000_pi8_key_t2v_5s_r0", "ok": true, "status": "success", "wall_s": 559.7, "exec_s": 557.0, "peak_vram_gb": 20.4, "outputs": ["g4000_pi8_key_t2v_5s_r0_00001_.mp4"], "error": null, "ts": "2026-08-06T17:58:17Z"}
{"label": "g4000_pi8_key_t2v_5s_r1", "task": "t2v", "model_file": "minimax_h3_fl2va_pruned_int8_convrot.safetensors", "prompt": "Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, waves crashing on dark rocks below, seagulls circling, warm sunlight flares, sound of surf and wind, gentle orchestral swell.", "width": 864, "height": 480, "seconds": 5, "length": 124, "steps": 20, "seed": 11, "sampler": "res_multistep", "scheduler": "simple", "image": "bench2_jp_01.jpg", "ref_image_size": "match", "ref_count": 1, "warm": true, "filename_prefix": "bench/g4000_pi8_key_t2v_5s_r1", "ok": true, "status": "success", "wall_s": 553.4, "exec_s": 551.4, "peak_vram_gb": 20.4, "outputs": ["g4000_pi8_key_t2v_5s_r1_00001_.mp4"], "error": null, "ts": "2026-08-06T18:07:31Z"}
{"label": "g4000_pi8_key_t2v_5s_r2", "task": "t2v", "model_file": "minimax_h3_fl2va_pruned_int8_convrot.safetensors", "prompt": "Cinematic aerial shot slowly orbiting a coastal lighthouse at golden hour, waves crashing on dark rocks below, seagulls circling, warm sunlight flares, sound of surf and wind, gentle orchestral swell.", "width": 864, "height": 480, "seconds": 5, "length": 124, "steps": 20, "seed": 12, "sampler": "res_multistep", "scheduler": "simple", "image": "bench2_jp_01.jpg", "ref_image_size": "match", "ref_count": 1, "warm": true, "filename_prefix": "bench/g4000_pi8_key_t2v_5s_r2", "ok": true, "status": "success", "wall_s": 553.4, "exec_s": 550.6, "peak_vram_gb": 20.4, "outputs": ["g4000_pi8_key_t2v_5s_r2_00001_.mp4"], "error": null, "ts": "2026-08-06T18:16:44Z"}
{"label": "g4000_pi8_key_ref2v_1ref_r0", "task": "ref2v", "model_file": "minimax_h3_ref2va_pruned_int8_convrot.safetensors", "prompt": "Use <Picture 1> as the setting. A slow cinematic pan across the sunset city skyline, clouds drifting, lights turning on in the towers, ambient city sounds.", "width": 864, "height": 480, "seconds": 5, "length": 124, "steps": 20, "seed": 10, "sampler": "res_multistep", "scheduler": "simple", "image": "bench2_jp_01.jpg", "ref_image_size": "match", "ref_count": 1, "warm": false, "filename_prefix": "bench/g4000_pi8_key_ref2v_1ref_r0", "ok": true, "status": "success", "wall_s": 589.8, "exec_s": 587.8, "peak_vram_gb": 19.0, "outputs": ["g4000_pi8_key_ref2v_1ref_r0_00001_.mp4"], "error": null, "ts": "2026-08-06T18:26:34Z"}
{"label": "g4000_pi8_key_ref2v_1ref_r1", "task": "ref2v", "model_file": "minimax_h3_ref2va_pruned_int8_convrot.safetensors", "prompt": "Use <Picture 1> as the setting. A slow cinematic pan across the sunset city skyline, clouds drifting, lights turning on in the towers, ambient city sounds.", "width": 864, "height": 480, "seconds": 5, "length": 124, "steps": 20, "seed": 11, "sampler": "res_multistep", "scheduler": "simple", "image": "bench2_jp_01.jpg", "ref_image_size": "match", "ref_count": 1, "warm": true, "filename_prefix": "bench/g4000_pi8_key_ref2v_1ref_r1", "ok": true, "status": "success", "wall_s": 582.4, "exec_s": 580.1, "peak_vram_gb": 19.0, "outputs": ["g4000_pi8_key_ref2v_1ref_r1_00001_.mp4"], "error": null, "ts": "2026-08-06T18:36:16Z"}
bench/4000/validation.jsonl
+6
View File
@@ -0,0 +1,6 @@
{"label": "g4000_smoke_t2v_b", "file": "g4000_smoke_t2v_b_00001_.mp4", "valid": true, "bytes": 619210, "video_codec": "h264", "audio_codec": "aac", "width": 608, "height": 352, "duration": 5.17, "expected_duration": 5.17}
{"label": "g4000_pi8_key_t2v_5s_r0", "file": "g4000_pi8_key_t2v_5s_r0_00001_.mp4", "valid": true, "bytes": 898728, "video_codec": "h264", "audio_codec": "aac", "width": 864, "height": 480, "duration": 5.17, "expected_duration": 5.17}
{"label": "g4000_pi8_key_t2v_5s_r1", "file": "g4000_pi8_key_t2v_5s_r1_00001_.mp4", "valid": true, "bytes": 861789, "video_codec": "h264", "audio_codec": "aac", "width": 864, "height": 480, "duration": 5.17, "expected_duration": 5.17}
{"label": "g4000_pi8_key_t2v_5s_r2", "file": "g4000_pi8_key_t2v_5s_r2_00001_.mp4", "valid": true, "bytes": 1325124, "video_codec": "h264", "audio_codec": "aac", "width": 864, "height": 480, "duration": 5.17, "expected_duration": 5.17}
{"label": "g4000_pi8_key_ref2v_1ref_r0", "file": "g4000_pi8_key_ref2v_1ref_r0_00001_.mp4", "valid": true, "bytes": 985435, "video_codec": "h264", "audio_codec": "aac", "width": 864, "height": 480, "duration": 5.17, "expected_duration": 5.17}
{"label": "g4000_pi8_key_ref2v_1ref_r1", "file": "g4000_pi8_key_ref2v_1ref_r1_00001_.mp4", "valid": true, "bytes": 974798, "video_codec": "h264", "audio_codec": "aac", "width": 864, "height": 480, "duration": 5.17, "expected_duration": 5.17}