storytold/cloud-worker
2
0
Fork 0
mirror of https://github.com/storytold/cloud-worker.git synced 2026-10-09 00:09:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

pod reset recovery: new SSH port, consolidated downloads to 1T volume, retire tmpfs overflow

Browse Source 
The network volume was resized to 1TB (which reset the pod and wiped the
container disk). All weights now live persistently on /workspace; the
/dev/shm overflow scheme is no longer needed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Hanashi
2026-08-06 01:58:41 -04:00
parent 548e4efcaf
commit 2d6dbf9f81
5 changed files with 82 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/local/port-forward.sh
+1 -1
View File
@@ -3,7 +3,7 @@
# Auto-reconnects if the tunnel drops. Ctrl-C to stop.
# Requires the echelon key loaded in your ssh agent.
POD_HOST=216.243.220.136
POD_PORT=14278
POD_PORT=17561
while true; do
ssh -N -L 8188:127.0.0.1:8188 \
-o ServerAliveInterval=30 -o ServerAliveCountMax=3 \
scripts/pod/download-weights-tmpfs.sh
-41
View File
@@ -1,41 +0,0 @@
#!/bin/bash
# Overflow download for the weight files that don't fit under the /workspace
# network-volume quota (~100GB on the current pod). Puts them on tmpfs
# (/dev/shm, 176GB on the B200 pod) — EPHEMERAL: lost on pod restart, so this
# is for benchmarking sessions. Re-run after any pod restart.
#
# tmux new -s downloads-shm -d 'bash /workspace/minimax-h3/download-weights-tmpfs.sh'
#
# ComfyUI sees this path via the minimax_shm section of extra_model_paths.yaml.
set -uo pipefail
REPO="Comfy-Org/MiniMax-H3"
DEST="/dev/shm/minimax-h3-models"
LOG="/workspace/minimax-h3/download.log"
mkdir -p "$DEST/diffusion_models"
export HF_HUB_ENABLE_HF_TRANSFER=1
log() { echo "[$(date -u +%FT%TZ)] $*" | tee -a "$LOG"; }
dl() {
local file="$1"
if [ -f "$DEST/$file" ]; then
log "SKIP $file (already present in shm)"
return 0
fi
log "START $file -> tmpfs"
if hf download "$REPO" "$file" --local-dir "$DEST" >>"$LOG" 2>&1; then
log "DONE $file ($(du -h "$DEST/$file" | cut -f1))"
else
log "FAILED $file"
return 1
fi
}
log "=== tmpfs overflow download starting ==="
dl diffusion_models/minimax_h3_fl2va_int8_convrot.safetensors
dl diffusion_models/minimax_h3_ref2va_bf16.safetensors
dl diffusion_models/minimax_h3_fl2va_bf16.safetensors
rm -rf "$DEST/.cache"
log "=== tmpfs overflow downloads complete ==="
scripts/pod/download-weights.sh
+1
View File
@@ -55,4 +55,5 @@ dl diffusion_models/minimax_h3_fl2va_int8_convrot.safetensors
dl diffusion_models/minimax_h3_ref2va_bf16.safetensors
dl diffusion_models/minimax_h3_fl2va_bf16.safetensors
rm -rf "$DEST/.cache"
log "=== All downloads complete ==="
scripts/pod/extra_model_paths.yaml
+2 -8
View File
@@ -1,13 +1,7 @@
# ComfyUI extra model paths — points at the RunPod network volume where all
# MiniMax H3 weights live. Install to /ComfyUI/extra_model_paths.yaml
# ComfyUI extra model paths — points at the RunPod network volume (1 TB) where
# all MiniMax H3 weights live. Install to /ComfyUI/extra_model_paths.yaml
minimax_workspace:
base_path: /workspace/minimax-h3/models
diffusion_models: diffusion_models
text_encoders: text_encoders
vae: vae
# tmpfs overflow for files that exceed the network-volume quota (~100GB).
# EPHEMERAL — refill with download-weights-tmpfs.sh after a pod restart.
minimax_shm:
base_path: /dev/shm/minimax-h3-models
diffusion_models: diffusion_models