mirror of
https://github.com/storytold/cloud-worker.git
synced 2026-10-09 00:09:43 +00:00
bench: survive server loss during polling (mark run failed, keep suite alive)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+13
-1
@@ -196,9 +196,21 @@ def run_one(api, cfg, poll=2.0, timeout=3600):
|
||||
prompt_id = resp["prompt_id"]
|
||||
exec_start = exec_end = None
|
||||
status_str = None
|
||||
poll_failures = 0
|
||||
while time.time() - t0 < timeout:
|
||||
time.sleep(poll)
|
||||
h = api.history(prompt_id)
|
||||
try:
|
||||
h = api.history(prompt_id)
|
||||
poll_failures = 0
|
||||
except Exception as e:
|
||||
poll_failures += 1
|
||||
if poll_failures >= 10:
|
||||
stop.set()
|
||||
return {**cfg, "ok": False, "status": "server_lost",
|
||||
"wall_s": round(time.time() - t0, 1), "exec_s": None,
|
||||
"peak_vram_gb": round(peak["vram"] / 1e9, 1) if peak["vram"] else None,
|
||||
"outputs": [], "error": f"server unreachable while polling: {e}"}
|
||||
continue
|
||||
if prompt_id in h:
|
||||
entry = h[prompt_id]
|
||||
status_str = entry.get("status", {}).get("status_str")
|
||||
|
||||
Reference in New Issue
Block a user