From 135e9ff50c5ad3f38cca649347f3a00141762eb9 Mon Sep 17 00:00:00 2001 From: saltacc Date: Thu, 1 Feb 2024 10:22:41 -0800 Subject: [PATCH 1/2] moving stuff around --- .gitmodules | 4 +- image_generation/ComfyLauncher/ComfyRunner.py | 21 ----- image_generation/ComfyLauncher/prompt.json | 86 ------------------- image_generation/ComfyUI | 1 - ..., eventually I'll have stuff in here..txt} | 0 workflows/comfy/ComfyLauncher/ComfyRunner.py | 68 +++++++++++++++ workflows/comfy/ComfyLauncher/README.md | 15 ++++ .../prompt/githubcantstoreemptyfolders | 0 ...t, eventually I'll have stuff in here..txt | 0 workflows/comfy/ComfyUI | 1 + .../comfy}/initialize_comfy_extensions.py | 0 .../comfy}/required_extensions.txt | 0 .../comfy}/required_requirements.txt | 0 13 files changed, 86 insertions(+), 110 deletions(-) delete mode 100644 image_generation/ComfyLauncher/ComfyRunner.py delete mode 100644 image_generation/ComfyLauncher/prompt.json delete mode 160000 image_generation/ComfyUI rename image_generation/{ComfyLauncher/README.md => web/extensions/FizzleDorf/Folder here to satisfy init, eventually I'll have stuff in here..txt} (100%) create mode 100644 workflows/comfy/ComfyLauncher/ComfyRunner.py create mode 100644 workflows/comfy/ComfyLauncher/README.md rename image_generation/ComfyLauncher/send_request.py => workflows/comfy/ComfyLauncher/prompt/githubcantstoreemptyfolders (100%) create mode 100644 workflows/comfy/ComfyLauncher/web/extensions/FizzleDorf/Folder here to satisfy init, eventually I'll have stuff in here..txt create mode 160000 workflows/comfy/ComfyUI rename {image_generation => workflows/comfy}/initialize_comfy_extensions.py (100%) rename {image_generation => workflows/comfy}/required_extensions.txt (100%) rename {image_generation => workflows/comfy}/required_requirements.txt (100%) diff --git a/.gitmodules b/.gitmodules index e5c085c..f100d87 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,6 +16,6 @@ [submodule "animation/rerender-lora/deps/gmflow"] path = animation/rerender-lora/deps/gmflow url = https://github.com/haofeixu/gmflow.git -[submodule "image_generation/ComfyUI"] - path = image_generation/ComfyUI +[submodule "workflows/comfy/ComfyUI"] + path = workflows/comfy/ComfyUI url = https://github.com/comfyanonymous/ComfyUI diff --git a/image_generation/ComfyLauncher/ComfyRunner.py b/image_generation/ComfyLauncher/ComfyRunner.py deleted file mode 100644 index 7d42b1b..0000000 --- a/image_generation/ComfyLauncher/ComfyRunner.py +++ /dev/null @@ -1,21 +0,0 @@ -import asyncio -import os -import sys - - -comfy_path = os.path.dirname(os.path.abspath(__file__)) -comfy_path = os.path.join(comfy_path, '..', 'ComfyUI') - -# change to ComfyUI directory -os.chdir(comfy_path) - -# add ComfyUI to path -sys.path.append(comfy_path) - -from server import PromptServer -from execution import PromptQueue - -loop = asyncio.new_event_loop() -asyncio.set_event_loop(loop) -server = PromptServer(loop) -q = PromptQueue(server) \ No newline at end of file diff --git a/image_generation/ComfyLauncher/prompt.json b/image_generation/ComfyLauncher/prompt.json deleted file mode 100644 index 5651f91..0000000 --- a/image_generation/ComfyLauncher/prompt.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "3": { - "class_type": "KSampler", - "inputs": { - "cfg": 8, - "denoise": 1, - "latent_image": [ - "5", - 0 - ], - "model": [ - "4", - 0 - ], - "negative": [ - "7", - 0 - ], - "positive": [ - "6", - 0 - ], - "sampler_name": "euler", - "scheduler": "normal", - "seed": 8566257, - "steps": 20 - } - }, - "4": { - "class_type": "CheckpointLoaderSimple", - "inputs": { - "ckpt_name": "v1-5-pruned-emaonly.ckpt" - } - }, - "5": { - "class_type": "EmptyLatentImage", - "inputs": { - "batch_size": 1, - "height": 512, - "width": 512 - } - }, - "6": { - "class_type": "CLIPTextEncode", - "inputs": { - "clip": [ - "4", - 1 - ], - "text": "masterpiece best quality girl" - } - }, - "7": { - "class_type": "CLIPTextEncode", - "inputs": { - "clip": [ - "4", - 1 - ], - "text": "bad hands" - } - }, - "8": { - "class_type": "VAEDecode", - "inputs": { - "samples": [ - "3", - 0 - ], - "vae": [ - "4", - 2 - ] - } - }, - "9": { - "class_type": "SaveImage", - "inputs": { - "filename_prefix": "ComfyUI", - "images": [ - "8", - 0 - ] - } - } -} \ No newline at end of file diff --git a/image_generation/ComfyUI b/image_generation/ComfyUI deleted file mode 160000 index 89507f8..0000000 --- a/image_generation/ComfyUI +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 89507f8adff4aff4507b6f35a67717badaecd4ac diff --git a/image_generation/ComfyLauncher/README.md b/image_generation/web/extensions/FizzleDorf/Folder here to satisfy init, eventually I'll have stuff in here..txt similarity index 100% rename from image_generation/ComfyLauncher/README.md rename to image_generation/web/extensions/FizzleDorf/Folder here to satisfy init, eventually I'll have stuff in here..txt diff --git a/workflows/comfy/ComfyLauncher/ComfyRunner.py b/workflows/comfy/ComfyLauncher/ComfyRunner.py new file mode 100644 index 0000000..141d381 --- /dev/null +++ b/workflows/comfy/ComfyLauncher/ComfyRunner.py @@ -0,0 +1,68 @@ +import asyncio +import json +import os +import sys +import threading +import signal + +import aiohttp + +current_script_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(current_script_dir, 'prompt', 'prompt.json') +prompt = json.load(open(file_path)) + +comfy_path = os.path.dirname(os.path.abspath(__file__)) +comfy_path = os.path.join(comfy_path, '..', 'ComfyUI') + +# change to ComfyUI directory +os.chdir(comfy_path) + +# add ComfyUI to path +sys.path.append(comfy_path) + + +from server import PromptServer +from execution import PromptQueue +from main import init_custom_nodes, cuda_malloc_warning, prompt_worker +import folder_paths + + +async def run(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + server = PromptServer(loop) + q = PromptQueue(server) + + init_custom_nodes() + + cuda_malloc_warning() + + server.add_routes() + + threading.Thread(target=prompt_worker, daemon=True, args=(q, server,)).start() + + folder_paths.add_model_folder_path("checkpoints", os.path.join(folder_paths.get_output_directory(), "checkpoints")) + folder_paths.add_model_folder_path("clip", os.path.join(folder_paths.get_output_directory(), "clip")) + folder_paths.add_model_folder_path("vae", os.path.join(folder_paths.get_output_directory(), "vae")) + + await server.start('127.0.0.1', 8188) + + async def send_prompt_async(): + # Load prompt + p = {"prompt": prompt} + data = json.dumps(p).encode('utf-8') + # Send HTTP post request + endpoint = "http://127.0.0.1:8188/prompt" + async with aiohttp.ClientSession() as session: + async with session.post(endpoint, data=data) as resp: + print( + await resp.text()) + + + await send_prompt_async() + + while server.prompt_queue.currently_running: + await asyncio.sleep(0.05) + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/workflows/comfy/ComfyLauncher/README.md b/workflows/comfy/ComfyLauncher/README.md new file mode 100644 index 0000000..a7d91b6 --- /dev/null +++ b/workflows/comfy/ComfyLauncher/README.md @@ -0,0 +1,15 @@ +## ComfyRunner + +Run a single ComfyUI request from the command line. + +### Usage + +Put the request parameters into `prompt/prompt.json` and run `ComfyRunner.py`. The ComfyUI folder must be named "ComfyUI" and be in the same superdirectory + +### Usage in backend + +It must run with the ComfyUI venv, example: +``` +COMFY_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/image_generation/ComfyUI" +COMFY_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python ../ComfyLauncher/ComfyLauncher.py" +``` diff --git a/image_generation/ComfyLauncher/send_request.py b/workflows/comfy/ComfyLauncher/prompt/githubcantstoreemptyfolders similarity index 100% rename from image_generation/ComfyLauncher/send_request.py rename to workflows/comfy/ComfyLauncher/prompt/githubcantstoreemptyfolders diff --git a/workflows/comfy/ComfyLauncher/web/extensions/FizzleDorf/Folder here to satisfy init, eventually I'll have stuff in here..txt b/workflows/comfy/ComfyLauncher/web/extensions/FizzleDorf/Folder here to satisfy init, eventually I'll have stuff in here..txt new file mode 100644 index 0000000..e69de29 diff --git a/workflows/comfy/ComfyUI b/workflows/comfy/ComfyUI new file mode 160000 index 0000000..c5a369a --- /dev/null +++ b/workflows/comfy/ComfyUI @@ -0,0 +1 @@ +Subproject commit c5a369a33ddb622827552716d9b0119035a2e666 diff --git a/image_generation/initialize_comfy_extensions.py b/workflows/comfy/initialize_comfy_extensions.py similarity index 100% rename from image_generation/initialize_comfy_extensions.py rename to workflows/comfy/initialize_comfy_extensions.py diff --git a/image_generation/required_extensions.txt b/workflows/comfy/required_extensions.txt similarity index 100% rename from image_generation/required_extensions.txt rename to workflows/comfy/required_extensions.txt diff --git a/image_generation/required_requirements.txt b/workflows/comfy/required_requirements.txt similarity index 100% rename from image_generation/required_requirements.txt rename to workflows/comfy/required_requirements.txt From 2a7c508d78a1629c29ddd82154f7a616271cbad0 Mon Sep 17 00:00:00 2001 From: saltacc Date: Fri, 2 Feb 2024 21:18:08 -0800 Subject: [PATCH 2/2] work with docker --- workflows/comfy/Dockerfile | 59 +++++++++++++++++++++++ workflows/comfy/README.md | 9 ++++ workflows/comfy/installer.sh | 10 ++++ workflows/comfy/required_requirements.txt | 1 + workflows/comfy/run_docker.sh | 18 +++++++ 5 files changed, 97 insertions(+) create mode 100644 workflows/comfy/Dockerfile create mode 100644 workflows/comfy/README.md create mode 100755 workflows/comfy/installer.sh create mode 100755 workflows/comfy/run_docker.sh diff --git a/workflows/comfy/Dockerfile b/workflows/comfy/Dockerfile new file mode 100644 index 0000000..eb3b0c4 --- /dev/null +++ b/workflows/comfy/Dockerfile @@ -0,0 +1,59 @@ + +# NB(bt, 2022-12-12): The version of containerized CUDA must match (or be less than?) that of the host system running this container. +# Juggling these dependencies, plus the underlying Ubuntu system (which controls Python versions), is tricky. +# Nvidia does not provide a CUDA 11.6 with Ubuntu 22.04, which mirrors my own desktop machine. Their CUDA 11.6 + Ubuntu 20.04 image has +# a stale Python (3.8), so we have to use the DeadSnakes PPA to install a more modern version. +# Gross. +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 + +# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md +LABEL org.opencontainers.image.title='Storyteller ML: Voice Conversion' +LABEL org.opencontainers.image.authors='adrien@storyteller.ai' + +# tzdata install hangs without presetting the time zone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Needed to install "future" release of Python 3.10 under Ubuntu 20.04, which only natively packages Python 3.8. +# We seem to be stuck on Ubuntu 22.04 because containerizd CUDA 11.6 is the only version we can run on my host machine, +# and Nvidia does not supply a (CUDA 11.6, Ubuntu 22.04) docker base image. +RUN apt-get update \ + && apt install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \ + curl \ + ffmpeg \ + libmagic-dev \ + libmagic1 \ + python3-pip \ + python3.10 \ + python3.10-venv \ + unzip \ + wget \ + git \ + --no-install-recommends + +RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser +# Copy files to container +COPY --chown=appuser:appgroup ./ComfyLauncher /app/ComfyLauncher +COPY --chown=appuser:appgroup ./ComfyUI /app/ComfyUI +COPY --chown=appuser:appgroup ./docker_requirements.txt /app/docker_requirements.txt +COPY --chown=appuser:appgroup ./required_requirements.txt /app/required_requirements.txt + +# install requirements +RUN curl -sS "https://bootstrap.pypa.io/get-pip.py" | python3.10 +# install known requirements for ComfyUI +RUN pip install -r /app/ComfyUI/requirements.txt +# install the detected requirements & ignore errors +RUN cat /app/docker_requirements.txt | xargs -n 1 -L 1 pip install || true +# install the extra requirements (manually written) +RUN pip install -r /app/required_requirements.txt + +RUN chmod -R 777 /app + +# set environment var +ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +USER appuser \ No newline at end of file diff --git a/workflows/comfy/README.md b/workflows/comfy/README.md new file mode 100644 index 0000000..5b511d8 --- /dev/null +++ b/workflows/comfy/README.md @@ -0,0 +1,9 @@ +# Docker Setup + +1. Run `installer.sh` (this will set up ComfyUI and create a docker_requirements.txt) +2. Run `docker build -t "storyteller/comfyui" . ` to create the docker container + +# Running the container + +A folder called `mounts` is where everything will be mounted into the container. Create it and set the +`COMFY_MOUNTS_DIRECTORY` environment variable in the `storyteller-rs` repo. \ No newline at end of file diff --git a/workflows/comfy/installer.sh b/workflows/comfy/installer.sh new file mode 100755 index 0000000..21fac4e --- /dev/null +++ b/workflows/comfy/installer.sh @@ -0,0 +1,10 @@ +python3 -m venv ComfyUI/venv +ComfyUI/venv/bin/pip install -r ComfyUI/requirements.txt +echo "ComfyUI has been installed successfully!" +ComfyUI/venv/bin/python3 initialize_comfy_extensions.py +echo "Extensions have been installed successfully!" +ComfyUI/venv/bin/pip freeze > docker_requirements.txt +echo "Requirements have been saved to docker_requirements.txt" +echo "Removing venv..." +rm -rf ComfyUI/venv +exit 0 diff --git a/workflows/comfy/required_requirements.txt b/workflows/comfy/required_requirements.txt index ecf9f9a..9299d3b 100644 --- a/workflows/comfy/required_requirements.txt +++ b/workflows/comfy/required_requirements.txt @@ -2,3 +2,4 @@ numexpr numba opencv-python color_matcher +mediapipe \ No newline at end of file diff --git a/workflows/comfy/run_docker.sh b/workflows/comfy/run_docker.sh new file mode 100755 index 0000000..fd94441 --- /dev/null +++ b/workflows/comfy/run_docker.sh @@ -0,0 +1,18 @@ +docker run --rm \ +--gpus all \ +--publish 8188:8188 \ +--user $(id -u):$(id -g) \ +-v $(pwd)/mounts/models/checkpoints:/app/ComfyUI/models/checkpoints \ +-v $(pwd)/mounts/models/loras:/app/ComfyUI/models/loras \ +-v $(pwd)/mounts/models/controlnet:/app/ComfyUI/models/controlnet \ +-v $(pwd)/mounts/models/ipadapter:/app/ComfyUI/models/ipadapter \ +-v $(pwd)/mounts/models/clip_vision:/app/ComfyUI/models/clip_vision \ +-v $(pwd)/mounts/prompt:/app/ComfyLauncher/prompt \ +-v $(pwd)/mounts/output:/app/ComfyUI/output \ +-v $(pwd)/mounts/input:/app/ComfyUI/input \ +-v $(pwd)/mounts/custom_nodes/ComfyUI-AnimateDiff-Evolved/models:/app/ComfyUI/custom_nodes/ComfyUI-AnimateDiff-Evolved/models \ +-v $(pwd)/mounts/custom_nodes/comfyui_controlnet_aux/ckpts:/app/ComfyUI/custom_nodes/comfyui_controlnet_aux/ckpts \ +-v $(pwd)/mounts/custom_nodes/ComfyUI-Frame-Interpolation/ckpts:/app/ComfyUI/custom_nodes/ComfyUI-Frame-Interpolation/ckpts \ +-it "storyteller/comfyui" python3 /app/ComfyLauncher/ComfyRunner.py + +# -it "test" /bin/bash \ No newline at end of file