storytold/cloud-worker
2
0
Fork 0
mirror of https://github.com/storytold/cloud-worker.git synced 2026-10-09 00:09:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

pod reset recovery: new SSH port, consolidated downloads to 1T volume, retire tmpfs overflow

Browse Source 
The network volume was resized to 1TB (which reset the pod and wiped the
container disk). All weights now live persistently on /workspace; the
/dev/shm overflow scheme is no longer needed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Hanashi
2026-08-06 01:58:41 -04:00
parent 548e4efcaf
commit 2d6dbf9f81
5 changed files with 82 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.claude/settings.local.json.tmp.3530818.d2f908315140
+78
View File
@@ -0,0 +1,78 @@
{
"permissions": {
"allow": [
"Bash(curl -sS -m 5 -o /dev/null -w \"HTTP %{http_code}\\\\n\" http://localhost:8188)",
"Bash(ssh *)",
"Bash(ssh-keygen -y -P \"\" -f ~/.ssh/id_ed25519_echelon)",
"Bash(ssh-add -l)",
"Bash(env)",
"Bash(echo \"exit: $?\")",
"Read(//run/user/1000/keyring/**)",
"Read(//tmp/**)",
"Bash(SSH_AUTH_SOCK=/run/user/1000/gnupg/S.gpg-agent.ssh ssh-add -l)",
"Bash(SSH_AUTH_SOCK=/run/user/1000/keyring/.ssh ssh-add -l)",
"Bash(echo \"exit2: $?\")",
"Bash(export SSH_AUTH_SOCK=/run/user/1000/keyring/.ssh)",
"Bash(curl -s \"https://huggingface.co/api/models/Comfy-Org/MiniMax-H3/tree/main?recursive=true\")",
"Bash(python3 -c ' *)",
"WebFetch(domain:github.com)",
"WebSearch",
"WebFetch(domain:huggingface.co)",
"Bash(curl -s \"https://huggingface.co/api/models/Comfy-Org/MiniMax-H3/tree/main/diffusion_models\")",
"WebFetch(domain:docs.comfy.org)",
"WebFetch(domain:blog.comfy.org)",
"WebFetch(domain:raw.githubusercontent.com)",
"WebFetch(domain:recipes.vllm.ai)",
"Bash(chmod +x scripts/pod/*.sh scripts/local/*.sh)",
"Bash(curl -sL https://raw.githubusercontent.com/Comfy-Org/ComfyUI/master/comfy_extras/nodes_minimax_h3.py -o nodes_minimax_h3.py)",
"WebFetch(domain:docs.sglang.io)",
"Bash(scp -P 14278 scripts/pod/download-weights.sh scripts/pod/start-comfyui.sh root@216.243.220.136:/workspace/minimax-h3/)",
"Bash(scp -P 14278 scripts/pod/extra_model_paths.yaml root@216.243.220.136:/ComfyUI/extra_model_paths.yaml)",
"WebFetch(domain:comfyui-wiki.com)",
"WebFetch(domain:docs.runpod.io)",
"Bash(curl -s \"https://api.github.com/repos/Comfy-Org/workflow_templates/git/trees/main?recursive=1\")",
"WebFetch(domain:kingy.ai)",
"Bash(bash /home/bt/dev/storyteller/cloud-worker/scripts/local/port-forward.sh)",
"Bash(curl -s -o /dev/null -w \"%{http_code}\" http://127.0.0.1:8188/)",
"Bash(python3 -)",
"Bash(curl -s http://127.0.0.1:8188/object_info)",
"Bash(curl -s \"https://api.github.com/search/issues?q=repo:Comfy-Org/ComfyUI+minimax+h3&per_page=30\")",
"Bash(curl -s http://127.0.0.1:8188/object_info/UNETLoader)",
"Bash(curl -s http://127.0.0.1:8188/object_info/VAELoader)",
"Bash(curl -s http://127.0.0.1:8188/object_info/CLIPLoader)",
"Bash(curl -s http://127.0.0.1:8188/object_info/CreateVideo)",
"Bash(python3 -m json.tool)",
"Bash(curl -s http://127.0.0.1:8188/object_info/SaveVideo)",
"Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); print\\(json.dumps\\(d['SaveVideo']['input'],indent=1\\)[:800]\\)\")",
"Bash(scp -q -O -P 14278 /home/bt/dev/storyteller/cloud-worker/scripts/pod/make-test-images.py root@216.243.220.136:/workspace/minimax-h3/)",
"Bash(timeout 550 python3 bench/minimax_bench.py --task t2v --width 608 --height 352 --seconds 5 --steps 20 --label smoke_t2v 2>&1)",
"Bash(timeout 550 python3 bench/minimax_bench.py --task i2v --width 608 --height 352 --seconds 5 --label smoke_i2v --warm)",
"Bash(timeout 550 python3 bench/minimax_bench.py --task ref2v --width 608 --height 352 --seconds 5 --label smoke_ref2v)",
"Bash(scp -q -O -P 14278 /home/bt/dev/storyteller/cloud-worker/scripts/pod/download-weights-tmpfs.sh root@216.243.220.136:/workspace/minimax-h3/)",
"Bash(scp -q -O -P 14278 /home/bt/dev/storyteller/cloud-worker/scripts/pod/extra_model_paths.yaml root@216.243.220.136:/ComfyUI/extra_model_paths.yaml)",
"Bash(ffmpeg -v error -i /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/scratchpad/smoke_t2v.mp4 -vf 'select=eq\\(n\\\\,60\\)' -frames:v 1 /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/scratchpad/t2v_frame.png -y)",
"Bash(ffmpeg -v error -i /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/scratchpad/smoke_ref2v.mp4 -vf 'select=eq\\(n\\\\,60\\)' -frames:v 1 /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/scratchpad/ref2v_frame.png -y)",
"Bash(python3 -c \"import ast; ast.parse\\(open\\('bench/minimax_bench.py'\\).read\\(\\)\\); print\\('syntax ok'\\)\")",
"Bash(python3 *)",
"Bash(git add *)",
"Bash(git commit -q -m 'MiniMax H3 on RunPod: install/download scripts, ComfyUI wiring, benchmark harness, research docs *)",
"Bash(tail -6 /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/tasks/betgtau6s.output)",
"Bash(git commit -q -m 'WEIGHTS.md: final download status — all 9 files present, none in progress *)",
"Bash(git commit -q -m 'docs: pad markdown table columns for raw-text readability *)",
"Bash(tail -4 /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/tasks/betgtau6s.output)",
"Bash(git commit -q -m 'bench: multi-reference ref2v support \\(1-8 ref images, match/max sizing\\) *)",
"Bash(curl -s http://127.0.0.1:8188/queue)",
"Bash(curl -s -o /dev/null -w \"local tunnel: %{http_code}\\\\n\" --max-time 5 http://127.0.0.1:8188/)",
"Bash(ls -la ~/.ssh/)",
"Bash(git rm *)",
"Bash(scp -q -O -P 17561 scripts/pod/download-weights.sh scripts/pod/start-comfyui.sh scripts/pod/make-test-images.py root@216.243.220.136:/workspace/minimax-h3/)",
"Bash(scp -q -O -P 17561 scripts/pod/extra_model_paths.yaml root@216.243.220.136:/ComfyUI/extra_model_paths.yaml)",
"Bash(curl -s -o /dev/null -w \"tunnel to ComfyUI: %{http_code}\\\\n\" http://127.0.0.1:8188/)",
"Bash(curl -s -o /dev/null -w \"tunnel: %{http_code}\\\\n\" --max-time 8 http://127.0.0.1:8188/)",
"Bash(curl -s --max-time 8 http://127.0.0.1:8188/object_info/UNETLoader)",
"Bash(pkill -f \"ssh -N -L 8188\")",
"Bash(kill 3568392)",
"Bash(git commit -q -m 'pod reset recovery: new SSH port, consolidated downloads to 1T volume, retire tmpfs overflow *)"
]
}
}
scripts/local/port-forward.sh
+1 -1
View File
@@ -3,7 +3,7 @@
# Auto-reconnects if the tunnel drops. Ctrl-C to stop.
# Requires the echelon key loaded in your ssh agent.
POD_HOST=216.243.220.136
POD_PORT=14278
POD_PORT=17561
while true; do
ssh -N -L 8188:127.0.0.1:8188 \
-o ServerAliveInterval=30 -o ServerAliveCountMax=3 \
scripts/pod/download-weights-tmpfs.sh
-41
View File
@@ -1,41 +0,0 @@
#!/bin/bash
# Overflow download for the weight files that don't fit under the /workspace
# network-volume quota (~100GB on the current pod). Puts them on tmpfs
# (/dev/shm, 176GB on the B200 pod) — EPHEMERAL: lost on pod restart, so this
# is for benchmarking sessions. Re-run after any pod restart.
#
# tmux new -s downloads-shm -d 'bash /workspace/minimax-h3/download-weights-tmpfs.sh'
#
# ComfyUI sees this path via the minimax_shm section of extra_model_paths.yaml.
set -uo pipefail
REPO="Comfy-Org/MiniMax-H3"
DEST="/dev/shm/minimax-h3-models"
LOG="/workspace/minimax-h3/download.log"
mkdir -p "$DEST/diffusion_models"
export HF_HUB_ENABLE_HF_TRANSFER=1
log() { echo "[$(date -u +%FT%TZ)] $*" | tee -a "$LOG"; }
dl() {
local file="$1"
if [ -f "$DEST/$file" ]; then
log "SKIP $file (already present in shm)"
return 0
fi
log "START $file -> tmpfs"
if hf download "$REPO" "$file" --local-dir "$DEST" >>"$LOG" 2>&1; then
log "DONE $file ($(du -h "$DEST/$file" | cut -f1))"
else
log "FAILED $file"
return 1
fi
}
log "=== tmpfs overflow download starting ==="
dl diffusion_models/minimax_h3_fl2va_int8_convrot.safetensors
dl diffusion_models/minimax_h3_ref2va_bf16.safetensors
dl diffusion_models/minimax_h3_fl2va_bf16.safetensors
rm -rf "$DEST/.cache"
log "=== tmpfs overflow downloads complete ==="
scripts/pod/download-weights.sh
+1
View File
@@ -55,4 +55,5 @@ dl diffusion_models/minimax_h3_fl2va_int8_convrot.safetensors
dl diffusion_models/minimax_h3_ref2va_bf16.safetensors
dl diffusion_models/minimax_h3_fl2va_bf16.safetensors
rm -rf "$DEST/.cache"
log "=== All downloads complete ==="
scripts/pod/extra_model_paths.yaml
+2 -8
View File
@@ -1,13 +1,7 @@
# ComfyUI extra model paths — points at the RunPod network volume where all
# MiniMax H3 weights live. Install to /ComfyUI/extra_model_paths.yaml
# ComfyUI extra model paths — points at the RunPod network volume (1 TB) where
# all MiniMax H3 weights live. Install to /ComfyUI/extra_model_paths.yaml
minimax_workspace:
base_path: /workspace/minimax-h3/models
diffusion_models: diffusion_models
text_encoders: text_encoders
vae: vae
# tmpfs overflow for files that exceed the network-volume quota (~100GB).
# EPHEMERAL — refill with download-weights-tmpfs.sh after a pod restart.
minimax_shm:
base_path: /dev/shm/minimax-h3-models
diffusion_models: diffusion_models