diff --git a/workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py b/workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py index ef5fa8a..ac4cb0c 100644 --- a/workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py +++ b/workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py @@ -3,67 +3,308 @@ import argparse import os import json import time +from typing import Dict, Any, Optional +from jsonpath_ng import jsonpath, parse +import shutil +from pathlib import Path +from enum import IntEnum, auto SERVER_IP = os.environ.get("SERVER_IP", "127.0.0.1:8188") TIMEOUT_SECONDS = int(os.environ.get("TIMEOUT_SECONDS", 1000)) PROMPT_ENDPOINT = f"http://{SERVER_IP}/prompt" +V2V_WORKFLOWS_DIRECTORY = os.environ.get("V2V_WORKFLOWS_DIRECTORY", + "/workflow_configs") +COMFY_ROOT_DIRECTORY = os.environ.get("COMFY_ROOT_DIRECTORY", + "/app/ComfyUI") + +MAIN_IPA_WORKFLOW = os.environ.get("MAIN_IPA_WORKFLOW", "ipa_API_03-05-2024/yae_vid2vid_main(+IPA)_03-05_API.json") +FACE_DETAILER_WORKFLOW = os.environ.get("FACE_DETAILER_WORKFLOW", "yae_vid2vid_FaceDetailer_03-05_API.json") +UPSCALER_WORKFLOW = os.environ.get("UPSCALER_WORKFLOW", "yae_vid2vid_Upscale_03-05_API.json") + + +class PipelineType(IntEnum): + BASE = auto() + IPA = auto() + FACE_DETAILER = auto() + UPSCALER = auto() + + + +style_to_filename = { + "anime_2_5d": "1_2.5d_anime_model.json", + "anime_2d_flat": "2_2d_flat_anime_model.json", + "cartoon_3d": "3_3d_cartoon_style.json", + "comic_book": "4_comic_book_model.json", + "anime_ghibli": "5_ghibli_anime_model.json", + "ink_punk": "6_ink_punk.json", + "ink_splash": "7_ink_splash.json", + "ink_bw_style": "8_ink_w_and_b_style.json", + "jojo_style": "9_jojo_style.json", + "paper_origami": "10_paper_origami.json", + "pixel_art": "11_pixel_art.json", + "pop_art": "12_pop_art.json", + "realistic_1": "13_realistic_1.json", + "realistic_2": "14_realistic_2.json", + "anime_retro_neon": "15_retro_neon_anime_90.json", + "anime_standard": "16_standard_anime_model.json", + "hr_giger": "17_hr_giger.json", + "simpsons": "18_simpsons.json", + "carnage": "19_carnage.json", + "pastel_cute_anime": "20_pastel_cute_anime.json", + "bloom_lighting": "21_bloom_lighting.json", + "25d_horror": "22_25D_Horror.json", + "creepy": "23_creepy.json", + "creepy_vhs": "24_creepy_vhs.json", + "trail_cam_footage": "25_trail_cam_footage.json", + "old_black_white_movie": "26_old_black_white_movie.json", + "horror_noir_black_white": "27_horror_noir_black_white.json", + "techno_noir_black_white": "28_techno_noir_black_white.json", + "black_white_20s": "29_black_white_20s.json", + "cyberpunk_anime": "30_cyberpunk_anime.json", + "dragonball": "31_dragonball.json", + "realistic_matrix": "32_realistic_matrix.json", + "realistic_cyberpunk": "33_realistic_cyberpunk.json" +} + + +def ensure_pipeline_input_present(): + left = Path(COMFY_ROOT_DIRECTORY) / "input/input.mp4" + right = Path(COMFY_ROOT_DIRECTORY) / "ComfyUI" / "input/input.mp4" + shutil.copy(left, right) + + +def convert_pipline_output_to_input(left): + right = Path(COMFY_ROOT_DIRECTORY) / "input/input.mp4" + shutil.copy(left, right) + + +def restore_last_pipeline_output(left): + right = Path(COMFY_ROOT_DIRECTORY) / "output" / "vid2vid/SparseUpscaleInterp_00001.mp4" + shutil.copy(left, right) + def parse_args(): parser = argparse.ArgumentParser(description='Run Comfy') - parser.add_argument('--prompt', type=str, help='location of the prompt json', required=True) + parser.add_argument('--prompt', type=str, help='location of the prompt json', required=False) + parser.add_argument('--style', type=str, help='style name', required=False) + parser.add_argument('--positive_prompt_filename', type=str, help='positive prompt', required=False) + parser.add_argument('--negative_prompt_filename', type=str, help='negative prompt', required=False) + parser.add_argument('--pause-between-steps', type=int, help='pause between steps', required=False, default=1) + parser.add_argument('--face-detailer-enabled', help='face detailer enabled', required=False, default=False, action=argparse.BooleanOptionalAction) + parser.add_argument('--upscaler-enabled', help='upscale enabled', required=False, default=False, action=argparse.BooleanOptionalAction) return parser.parse_args() args = parse_args() -# load prompt json -file_path = args.prompt -print(f'Loading prompt json from path: {file_path}') -prompt = json.load(open(file_path)) - -# check the health of the server (5 second timeout) -response = requests.get(PROMPT_ENDPOINT, timeout=5) -if response.status_code == 200: - print("Server is healthy") -else: - print("Server is unhealthy") - print(f"Status code: {response.status_code}") - print(f"Response: {response.text}") - -# make sure there are no jobs already -queue_json = response.json() -if queue_json["exec_info"]["queue_remaining"]: - print("There are jobs in the queue already. Exiting...") - exit(1) -else: - print("Queue is empty") - - -# Load prompt -p = {"prompt": prompt} -data = json.dumps(p).encode('utf-8') -# Send HTTP post request -response = requests.post(PROMPT_ENDPOINT, data=data) -if response.status_code == 200: - print("Job enqueued successfully") -else: - print("Job failed to queue") - print(f"Status code: {response.status_code}") - print(f"Response: {response.text}") - exit(1) - -# Wait for the job to finish -print(f"Waiting for job to finish (timeout: {TIMEOUT_SECONDS} seconds)") -start_time = time.time() -# check queue endpoint in loop -while True: +def ensure_server_is_healthy(): + # check the health of the server (5 second timeout) response = requests.get(PROMPT_ENDPOINT, timeout=5) + if response.status_code == 200: + print("Server is healthy") + else: + print("Server is unhealthy") + print(f"Status code: {response.status_code}") + print(f"Response: {response.text}") + + # make sure there are no jobs already queue_json = response.json() if queue_json["exec_info"]["queue_remaining"]: - pass - else: - print("Job is done") - break - if time.time() - start_time > TIMEOUT_SECONDS: - print("Timeout reached and job didn't finish!") + print("There are jobs in the queue already. Exiting...") exit(1) - time.sleep(1) + else: + print("Queue is empty") + +def main(): + ensure_server_is_healthy() + + pipeline_type = PipelineType.BASE + if args.face_detailer_enabled or args.upscaler_enabled: + pipeline_type = PipelineType.IPA + + output_index = 0 + file_path = args.prompt + prompt = None + if args.style is not None: + print("Generating prompt json") + style = args.style + positive_prompt_filename = args.positive_prompt_filename + negative_prompt_filename = args.negative_prompt_filename + positive_prompt = None + negative_prompt = None + workflow_filename = None + if pipeline_type == PipelineType.IPA: + workflow_filename = MAIN_IPA_WORKFLOW + if positive_prompt_filename: + positive_prompt = open(positive_prompt_filename).read() + if negative_prompt_filename: + negative_prompt = open(negative_prompt_filename).read() + validate_style_name(style) + print("running pipeline" + str(pipeline_type)) + # ensure_pipeline_input_present() + prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename) + enqueue_prompt_and_wait(prompt) + output_index = output_index + 1 + left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4" + if args.upscaler_enabled: + # time.sleep(30) + print("running pipeline" + str(pipeline_type)) + convert_pipline_output_to_input(left) + workflow_filename = UPSCALER_WORKFLOW + prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.UPSCALER) + enqueue_prompt_and_wait(prompt) + output_index = output_index + 1 + left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4" + # time.sleep(30) + if args.face_detailer_enabled: + # time.sleep(30) + print("running pipeline" + str(pipeline_type)) + convert_pipline_output_to_input(left) + workflow_filename = FACE_DETAILER_WORKFLOW + prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.FACE_DETAILER) + enqueue_prompt_and_wait(prompt) + output_index = output_index + 1 + left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4" + # time.sleep(30) + restore_last_pipeline_output(left) + else: + print(f'Loading prompt json from path: {file_path}') + prompt_workflow = json.load(open(file_path)) + prompt = {"prompt": prompt_workflow} + enqueue_prompt_and_wait(prompt) + +def enqueue_prompt_and_wait(prompt): + data = json.dumps(prompt).encode('utf-8') + # Send HTTP post request + response = requests.post(PROMPT_ENDPOINT, data=data) + if response.status_code == 200: + print("Job enqueued successfully") + else: + print("Job failed to queue") + print(f"Status code: {response.status_code}") + print(f"Response: {response.text}") + exit(1) + + # Wait for the job to finish + print(f"Waiting for job to finish (timeout: {TIMEOUT_SECONDS} seconds)") + start_time = time.time() + # check queue endpoint in loop + while True: + response = requests.get(PROMPT_ENDPOINT, timeout=5) + queue_json = response.json() + if queue_json["exec_info"]["queue_remaining"]: + pass + else: + print("Job is done") + break + if time.time() - start_time > TIMEOUT_SECONDS: + print("Timeout reached and job didn't finish!") + exit(1) + time.sleep(1) + + + +def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, workflow_filename=None, pipeline_type = PipelineType.BASE) -> Path: + styles_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "styles" + workflow_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "workflows" + mappings_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "mappings" + + style_filename = style_to_filename[style_name] + style_json = json.load(open(styles_directory / style_filename)) + + workflow_json_filename = style_json["workflow_api_name"] + mapping_json_filename = "mapping_one.json" + + if workflow_filename: + workflow_json_filename = workflow_filename + + workflow_json = json.load(open(workflow_directory / workflow_json_filename)) + mapping_json = json.load(open(mappings_directory / mapping_json_filename)) + + json_mods = style_json["modifications"] + jsonpath_mods = get_jsonpath_mods(json_mods, mapping_json, pos_in=positive_prompt, neg_in=negative_prompt, pipeline_type=pipeline_type) + print(*jsonpath_mods.items(), sep="\n") + + json_prompt = apply_jsonpath_mods(jsonpath_mods, workflow_json) + + p = {"prompt": json_prompt} + return p + + +def get_jsonpath_mods(style_mods: Dict[str, Any], mapping_json: Dict[str, Any], pos_in: Optional[str] = None, + neg_in: Optional[str] = None, pipeline_type = PipelineType.BASE) -> Dict[str, Any]: + modifications = {} + new_mod_json = style_mods.copy() + # Process "loras" differently + loras = new_mod_json.get("loras", []) + if len(loras) > 8: + raise ValueError("Too many loras, max is 8") + + for index, lora in enumerate(loras): + if "name" in lora and "strength" in lora: + lora_strength = lora["strength"] + if pipeline_type == PipelineType.FACE_DETAILER: + lora_strength = 0.5 + new_mod_json[f"lora_{index + 1}_strength"] = lora_strength + new_mod_json[f"lora_{index + 1}_name"] = lora["name"] + + # Exclude "loras" from further processing + new_mod_json.pop("loras", None) + + # Handling positive and negative prompts + if pos_in: + new_mod_json["positive_prompt"] = f"{new_mod_json.get('positive_prompt', '')}, {pos_in}" + if neg_in: + new_mod_json["negative_prompt"] = f"{new_mod_json.get('negative_prompt', '')}, {neg_in}" + + for key, value in new_mod_json.items(): + mapping_key = f"$.{key}" + jsonpath_expr = parse(mapping_key) + + # Finding matches using jsonpath_ng + matches = [match.value for match in jsonpath_expr.find(mapping_json)] + if matches: + mapping_value = matches[0] + modifications[mapping_value] = value + else: + print(f"No mapping found for key '{key}'") + + return modifications + + +def apply_jsonpath_mods(jsonpath_mods: Dict[str, Any], workflow_json: Dict[str, Any]) -> Dict[str, Any]: + for key, value in jsonpath_mods.items(): + # jsonpath-ng expects numbers to be wrapped in quotes + parts = key.split(".") + new_parts = [f"'{part}'" if part.isdigit() else part for part in parts] + key = ".".join(new_parts) + + jsonpath_expr = parse(key) + matches = jsonpath_expr.find(workflow_json) + + if matches: + parent = matches[0].context.value + + if isinstance(parent, dict): + last_part = new_parts[-1].strip("'") + parent[last_part] = value + + elif isinstance(parent, list): + index = int(new_parts[-1].strip("'")) + if 0 <= index < len(parent): + parent[index] = value + else: + print(f"Index {index} is out of bounds for key '{key}'") + else: + print(f"No mapping found for key '{key}'") + + return workflow_json + + +def validate_style_name(style): + if style not in style_to_filename: + print(f"Invalid style: {args.style}") + print(f"Valid styles: {', '.join(style_to_filename.keys())}") + exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/workflows/comfy/Dockerfile b/workflows/comfy/Dockerfile index eaf229c..7682b72 100644 --- a/workflows/comfy/Dockerfile +++ b/workflows/comfy/Dockerfile @@ -57,15 +57,13 @@ COPY ./pip.conf /root/.config/pip/pip.conf WORKDIR /app # Copy files to container -COPY --chown=appuser:appgroup initialize_comfy_extensions.py /app/initialize_comfy_extensions.py -COPY --chown=appuser:appgroup ComfyLauncher /app/ComfyLauncher -COPY --chown=appuser:appgroup format_video.py /app/format_video.py + COPY --chown=appuser:appgroup install.py /app/install.py COPY --chown=appuser:appgroup list-repos.sh /app/list-repos.sh RUN chmod -R 777 /app # download & install comfy ui and its dependencies -RUN python3 install.py --comfy-ui-url https://github.com/comfyanonymous/ComfyUI.git --comfy-ui-commit eda87043862f743b0a0467735f8531f7c4709b3a +RUN python3 install.py --comfy-ui-url https://github.com/comfyanonymous/ComfyUI.git --comfy-ui-commit ece5acb8e8025d8ca26aa880f604d971d245475d RUN python3 -m venv ComfyUI/venv RUN ComfyUI/venv/bin/python -m pip install -r ComfyUI/requirements.txt --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/ && \ ComfyUI/venv/bin/python -m pip cache purge @@ -82,5 +80,9 @@ RUN python3 install.py --repos-json repos.json COPY --chown=appuser:appgroup dependencies.json /app/dependencies.json COPY --chown=appuser:appgroup server_requirements.txt /app/server_requirements.txt +COPY --chown=appuser:appgroup initialize_comfy_extensions.py /app/initialize_comfy_extensions.py +COPY --chown=appuser:appgroup ComfyLauncher /app/ComfyLauncher +COPY --chown=appuser:appgroup format_video.py /app/format_video.py + # protobuf use python implementation (weird error without) ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python diff --git a/workflows/comfy/repos.json b/workflows/comfy/repos.json index b6b55d2..f039a1e 100644 --- a/workflows/comfy/repos.json +++ b/workflows/comfy/repos.json @@ -6,22 +6,22 @@ }, { "folder": "ComfyUI/custom_nodes/ComfyUI_FizzNodes", - "commit": "2ad19c1c3ad424b732fa0bc16ffb02284676a5da", + "commit": "0e05436071d18db18900a6e796c1358a4e9b10a3", "url": "https://github.com/FizzleDorf/ComfyUI_FizzNodes.git" }, { "folder": "ComfyUI/custom_nodes/ComfyUI-VideoHelperSuite", - "commit": "f94597739f20dac331f9918a245149b6b00a60f2", + "commit": "47e56f3e42c3ab4fb33f1407c2c40b5391226621", "url": "https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git" }, { "folder": "ComfyUI/custom_nodes/ComfyUI-Custom-Scripts", - "commit": "b4a9cebf24b50c4b538945ae86c17a9dc24052a8", + "commit": "3f2c021e50be2fed3c9d1552ee8dcaae06ad1fe5", "url": "https://github.com/pythongosssss/ComfyUI-Custom-Scripts.git" }, { "folder": "ComfyUI/custom_nodes/ComfyUI-Advanced-ControlNet", - "commit": "98768c7c3c45ac1e803071c5ad8e675e360e9e73", + "commit": "33d9884b76e8d7a2024691c5d98308e7e61bf38d", "url": "https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet.git" }, { @@ -31,22 +31,22 @@ }, { "folder": "ComfyUI/custom_nodes/ComfyUI-AnimateDiff-Evolved", - "commit": "9ccea911a63e2119778fa91b2692a8470a48e9da", + "commit": "b693c681731202152cd8dd45714bb7bc34a4ca6b", "url": "https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved.git" }, { "folder": "ComfyUI/custom_nodes/ComfyUI-Impact-Pack", - "commit": "e374bd304a4a5b44d0deae2b2afaf4d5faad4810", + "commit": "a7076b39685bcb3bf9d5c7180f5e538839c2ccb0", "url": "https://github.com/ltdrdata/ComfyUI-Impact-Pack.git" }, { "folder": "ComfyUI/custom_nodes/ComfyUI-Video-Matting", - "commit": "178bf956ddb817d065b1b073ec04582ad60335bb", + "commit": "940fcc5a56d4b47a6c3507d827c56b083c388f3e", "url": "https://github.com/Fannovel16/ComfyUI-Video-Matting.git" }, { "folder": "ComfyUI/custom_nodes/comfyui_segment_anything", - "commit": "681d977c2cc244de729fd9e08145ff365c609262", + "commit": "ab6395596399d5048639cdab7e44ec9fae857a93", "url": "https://github.com/storyicon/comfyui_segment_anything.git" }, { @@ -56,7 +56,7 @@ }, { "folder": "ComfyUI/custom_nodes/ComfyUI_IPAdapter_plus", - "commit": "6a411dcb2c6c3b91a3aac97adfb080a77ade7d38", + "commit": "20125bf9394b1bc98ef3228277a31a3a52c72fc2", "url": "https://github.com/cubiq/ComfyUI_IPAdapter_plus.git" }, { @@ -66,32 +66,32 @@ }, { "folder": "ComfyUI/custom_nodes/ComfyUI_essentials", - "commit": "0283ce45272fc0737e3420c0aa2c5c30a74f7a36", + "commit": "84d25b99c2e3e82142c2de6656fcda55e5ce1318", "url": "https://github.com/cubiq/ComfyUI_essentials.git" }, { "folder": "ComfyUI/custom_nodes/efficiency-nodes-comfyui", - "commit": "f6d01dc5449e2cc6e29da92b3b337e49c0dce112", + "commit": "3b7e89d969c02b80ff3662f34813679167b835fa", "url": "https://github.com/jags111/efficiency-nodes-comfyui.git" }, { "folder": "ComfyUI/custom_nodes/rgthree-comfy", - "commit": "9021670e9ff887d40e031334fc9f3e92c599d161", + "commit": "017a1595cdfb75d34d91f4d8abceab89202d0c81", "url": "https://github.com/rgthree/rgthree-comfy.git" }, { "folder": "ComfyUI/custom_nodes/ComfyUI-Frame-Interpolation", - "commit": "5e11679995c68f33891c306a393915feefe234b5", + "commit": "1c4c4b4f9a99e7e6eb7c5a5f3fdc7c9dfd319357", "url": "https://github.com/Fannovel16/ComfyUI-Frame-Interpolation.git" }, { "folder": "ComfyUI/custom_nodes/comfyui_controlnet_aux", - "commit": "33d6e86b65c40e5f97052262c6900f413684ccde", + "commit": "ac8702cc38feb6cf3fe4f8e11d2792a86cb56cc4", "url": "https://github.com/Fannovel16/comfyui_controlnet_aux.git" }, { "folder": "ComfyUI/custom_nodes/ComfyUI-Manager", - "commit": "6c553d1228978352767f4914c4dd7787b00f6769", + "commit": "1758f4b3dfed65db813a52cc87d72dcd5088d1fb", "url": "https://github.com/ltdrdata/ComfyUI-Manager.git" }, { @@ -106,17 +106,27 @@ }, { "folder": "ComfyUI/custom_nodes/ComfyUI-Crystools", - "commit": "8c257748b98f59357ac20cebd4b4b67c0a9e53fe", + "commit": "bfbfd121a245d04ce7d06ccc36acdb9873c47d9d", "url": "https://github.com/crystian/ComfyUI-Crystools.git" }, { "folder": "ComfyUI/custom_nodes/ComfyUI-KJNodes", - "commit": "df86fd77ac7f832b35ddaa748e4cf421ce9bf9e1", + "commit": "b2d5ab3fcdc81c807f4049255abb65d38b608992", "url": "https://github.com/kijai/ComfyUI-KJNodes.git" }, { "folder": "ComfyUI/custom_nodes/comfy_mtb", - "commit": "54ff6583ded0ed4054f8e5d7fadf0b2350259dce", + "commit": "f9d2ebf91d09fc214fecf7501a5490b33c30aca2", "url": "https://github.com/melMass/comfy_mtb.git" + }, + { + "folder": "ComfyUI/custom_nodes/comfyui-art-venture", + "commit": "c33412288fbdcd132265c9029a38001fd9696aa5", + "url": "https://github.com/sipherxyz/comfyui-art-venture.git" + }, + { + "folder": "ComfyUI/custom_nodes/ComfyMath", + "commit": "be9beab9923ccf5c5e4132dc1653bcdfa773ed70", + "url": "https://github.com/evanspearman/ComfyMath" } ] \ No newline at end of file diff --git a/workflows/comfy/required_requirements.txt b/workflows/comfy/required_requirements.txt index 06d6657..eb901c9 100644 --- a/workflows/comfy/required_requirements.txt +++ b/workflows/comfy/required_requirements.txt @@ -12,3 +12,4 @@ addict yapf deepdiff pynvml +pydantic \ No newline at end of file