storytold/storyteller-ml
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-ml.git synced 2026-10-25 08:40:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

live portraits: more resilient to comfy errors

Browse Source
This commit is contained in:
Kasisnu
2024-08-08 18:05:19 +05:30
parent e672768c6f
commit 9d2b09b492
2 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workflows/comfy/ComfyLauncher/ComfyLivePortraitRunner.py
+8 -4
View File
@@ -18,7 +18,7 @@ import uuid
client_id = str(uuid.uuid4()) client_id = str(uuid.uuid4())
SERVER_IP = os.environ.get("SERVER_IP", "127.0.0.1:8188") SERVER_IP = os.environ.get("SERVER_IP", "127.0.0.1:8188")
SERVER_START_TIMEOUT = int(os.environ.get("SERVER_START_TIMEOUT", 120)) SERVER_START_TIMEOUT = int(os.environ.get("SERVER_START_TIMEOUT", 60))
TIMEOUT_SECONDS = int(os.environ.get("TIMEOUT_SECONDS", 180)) TIMEOUT_SECONDS = int(os.environ.get("TIMEOUT_SECONDS", 180))
PROMPT_ENDPOINT = f"http://{SERVER_IP}/prompt" PROMPT_ENDPOINT = f"http://{SERVER_IP}/prompt"
@@ -114,7 +114,7 @@ def trigger_comfy_restart(restart_trigger_file):
print("Restart triggered") print("Restart triggered")
# KS: we're making the retries configurable but we should not increase the retries for all workflows without confirming we'd still meet slas. Some jobs are better to just fail. Retries compound over inference-job attempt_counts, comfy startup times, and actual inference times. # KS: we're making the retries configurable but we should not increase the retries for all workflows without confirming we'd still meet slas. Some jobs are better to just fail. Retries compound over inference-job attempt_counts, comfy startup times, and actual inference times.
def ensure_server_is_healthy(server_address, restart_trigger_file=None, retries=1): def ensure_server_is_healthy(server_address, restart_trigger_file=None, retries=2):
wait_for_comfy_startup(server_address) wait_for_comfy_startup(server_address)
if queued_count_is_zero(server_address): if queued_count_is_zero(server_address):
print("Server is healthy") print("Server is healthy")
@@ -134,12 +134,14 @@ def ensure_server_is_healthy(server_address, restart_trigger_file=None, retries=
print(f"Server is not healthy. Number of queued jobs: {number_of_queued_jobs}") print(f"Server is not healthy. Number of queued jobs: {number_of_queued_jobs}")
print("Server is not healthy after restart. Retrying restart") print("Server is not healthy after restart. Retrying restart")
wait_for_comfy_startup(server_address, timeout=5, terminate_on_failure=True)
def interrupt_comfy_queue(server_address): def interrupt_comfy_queue(server_address):
requests.post(f"http://{server_address}/interrupt") requests.post(f"http://{server_address}/interrupt")
print("Queue interrupted successfully.") print("Queue interrupted successfully.")
def wait_for_comfy_startup(server_address, timeout=SERVER_START_TIMEOUT): def wait_for_comfy_startup(server_address, timeout=SERVER_START_TIMEOUT, terminate_on_failure=False):
prompt_endpoint = f"http://{server_address}/prompt" prompt_endpoint = f"http://{server_address}/prompt"
start_time = time.time() start_time = time.time()
started = False started = False
@@ -156,7 +158,9 @@ def wait_for_comfy_startup(server_address, timeout=SERVER_START_TIMEOUT):
time.sleep(1) time.sleep(1)
if not started: if not started:
print(f"Server failed to start in {SERVER_START_TIMEOUT} seconds") print(f"Server failed to start in {SERVER_START_TIMEOUT} seconds")
exit(1) if terminate_on_failure:
print("Exiting.")
exit(1)
async def main(): async def main():
workflows/comfy/live-runner-comfy-server-starter.sh
+1 -1
View File
@@ -8,4 +8,4 @@ source /app/venv/bin/activate
date > /restart.txt date > /restart.txt
cd /app/ComfyUI cd /app/ComfyUI
echo /restart.txt | entr -nrz timeout -k 5 0 python main.py --listen 0.0.0.0 --highvram --disable-metadata echo /restart.txt | entr -nr timeout -k 5 0 python main.py --listen 0.0.0.0 --highvram --disable-metadata