Merge pull request #39 from storytold/feature/comfyui

Docker support, move to comfy folder

.gitmodules

@@ -16,6 +16,6 @@
 [submodule "animation/rerender-lora/deps/gmflow"]
 	path = animation/rerender-lora/deps/gmflow
 	url = https://github.com/haofeixu/gmflow.git
-[submodule "image_generation/ComfyUI"]
+[submodule "workflows/comfy/ComfyUI"]
-	path = image_generation/ComfyUI
+	path = workflows/comfy/ComfyUI
 	url = https://github.com/comfyanonymous/ComfyUI

image_generation/ComfyLauncher/ComfyRunner.py

@@ -1,21 +0,0 @@
-import asyncio
-import os
-import sys
-
-
-comfy_path = os.path.dirname(os.path.abspath(__file__))
-comfy_path = os.path.join(comfy_path, '..', 'ComfyUI')
-
-# change to ComfyUI directory
-os.chdir(comfy_path)
-
-# add ComfyUI to path
-sys.path.append(comfy_path)
-
-from server import PromptServer
-from execution import PromptQueue
-
-loop = asyncio.new_event_loop()
-asyncio.set_event_loop(loop)
-server = PromptServer(loop)
-q = PromptQueue(server)

image_generation/ComfyLauncher/prompt.json

@@ -1,86 +0,0 @@
-{
-    "3": {
-        "class_type": "KSampler",
-        "inputs": {
-            "cfg": 8,
-            "denoise": 1,
-            "latent_image": [
-                "5",
-                0
-            ],
-            "model": [
-                "4",
-                0
-            ],
-            "negative": [
-                "7",
-                0
-            ],
-            "positive": [
-                "6",
-                0
-            ],
-            "sampler_name": "euler",
-            "scheduler": "normal",
-            "seed": 8566257,
-            "steps": 20
-        }
-    },
-    "4": {
-        "class_type": "CheckpointLoaderSimple",
-        "inputs": {
-            "ckpt_name": "v1-5-pruned-emaonly.ckpt"
-        }
-    },
-    "5": {
-        "class_type": "EmptyLatentImage",
-        "inputs": {
-            "batch_size": 1,
-            "height": 512,
-            "width": 512
-        }
-    },
-    "6": {
-        "class_type": "CLIPTextEncode",
-        "inputs": {
-            "clip": [
-                "4",
-                1
-            ],
-            "text": "masterpiece best quality girl"
-        }
-    },
-    "7": {
-        "class_type": "CLIPTextEncode",
-        "inputs": {
-            "clip": [
-                "4",
-                1
-            ],
-            "text": "bad hands"
-        }
-    },
-    "8": {
-        "class_type": "VAEDecode",
-        "inputs": {
-            "samples": [
-                "3",
-                0
-            ],
-            "vae": [
-                "4",
-                2
-            ]
-        }
-    },
-    "9": {
-        "class_type": "SaveImage",
-        "inputs": {
-            "filename_prefix": "ComfyUI",
-            "images": [
-                "8",
-                0
-            ]
-        }
-    }
-}

workflows/comfy/ComfyLauncher/ComfyRunner.py

@@ -0,0 +1,68 @@
+import asyncio
+import json
+import os
+import sys
+import threading
+import signal
+
+import aiohttp
+
+current_script_dir = os.path.dirname(os.path.realpath(__file__))
+file_path = os.path.join(current_script_dir, 'prompt', 'prompt.json')
+prompt = json.load(open(file_path))
+
+comfy_path = os.path.dirname(os.path.abspath(__file__))
+comfy_path = os.path.join(comfy_path, '..', 'ComfyUI')
+
+# change to ComfyUI directory
+os.chdir(comfy_path)
+
+# add ComfyUI to path
+sys.path.append(comfy_path)
+
+
+from server import PromptServer
+from execution import PromptQueue
+from main import init_custom_nodes, cuda_malloc_warning, prompt_worker
+import folder_paths
+
+
+async def run():
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    server = PromptServer(loop)
+    q = PromptQueue(server)
+
+    init_custom_nodes()
+
+    cuda_malloc_warning()
+
+    server.add_routes()
+
+    threading.Thread(target=prompt_worker, daemon=True, args=(q, server,)).start()
+
+    folder_paths.add_model_folder_path("checkpoints", os.path.join(folder_paths.get_output_directory(), "checkpoints"))
+    folder_paths.add_model_folder_path("clip", os.path.join(folder_paths.get_output_directory(), "clip"))
+    folder_paths.add_model_folder_path("vae", os.path.join(folder_paths.get_output_directory(), "vae"))
+
+    await server.start('127.0.0.1', 8188)
+
+    async def send_prompt_async():
+        # Load prompt
+        p = {"prompt": prompt}
+        data = json.dumps(p).encode('utf-8')
+        # Send HTTP post request
+        endpoint = "http://127.0.0.1:8188/prompt"
+        async with aiohttp.ClientSession() as session:
+            async with session.post(endpoint, data=data) as resp:
+                print(
+                    await resp.text())
+
+
+    await send_prompt_async()
+
+    while server.prompt_queue.currently_running:
+        await asyncio.sleep(0.05)
+
+if __name__ == "__main__":
+    asyncio.run(run())

workflows/comfy/ComfyLauncher/README.md

@@ -0,0 +1,15 @@
+## ComfyRunner
+
+Run a single ComfyUI request from the command line.
+
+### Usage
+
+Put the request parameters into `prompt/prompt.json` and run `ComfyRunner.py`. The ComfyUI folder must be named "ComfyUI" and be in the same superdirectory
+
+### Usage in backend
+
+It must run with the ComfyUI venv, example:
+```
+COMFY_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/image_generation/ComfyUI"
+COMFY_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python ../ComfyLauncher/ComfyLauncher.py"
+```

workflows/comfy/Dockerfile

@@ -0,0 +1,59 @@
+
+# NB(bt, 2022-12-12): The version of containerized CUDA must match (or be less than?) that of the host system running this container.
+#  Juggling these dependencies, plus the underlying Ubuntu system (which controls Python versions), is tricky.
+#  Nvidia does not provide a CUDA 11.6 with Ubuntu 22.04, which mirrors my own desktop machine. Their CUDA 11.6 + Ubuntu 20.04 image has
+#  a stale Python (3.8), so we have to use the DeadSnakes PPA to install a more modern version.
+#  Gross.
+FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
+
+# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md
+LABEL org.opencontainers.image.title='Storyteller ML: Voice Conversion'
+LABEL org.opencontainers.image.authors='adrien@storyteller.ai'
+
+# tzdata install hangs without presetting the time zone
+ENV TZ=UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Needed to install "future" release of Python 3.10 under Ubuntu 20.04, which only natively packages Python 3.8.
+# We seem to be stuck on Ubuntu 22.04 because containerizd CUDA 11.6 is the only version we can run on my host machine,
+# and Nvidia does not supply a (CUDA 11.6, Ubuntu 22.04) docker base image.
+RUN apt-get update \
+    && apt install software-properties-common -y \
+    && add-apt-repository ppa:deadsnakes/ppa
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \
+        curl \
+        ffmpeg \
+        libmagic-dev \
+        libmagic1 \
+        python3-pip \
+        python3.10 \
+        python3.10-venv \
+        unzip \
+        wget \
+        git \
+        --no-install-recommends
+
+RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser
+# Copy files to container
+COPY --chown=appuser:appgroup ./ComfyLauncher /app/ComfyLauncher
+COPY --chown=appuser:appgroup ./ComfyUI /app/ComfyUI
+COPY --chown=appuser:appgroup ./docker_requirements.txt /app/docker_requirements.txt
+COPY --chown=appuser:appgroup ./required_requirements.txt /app/required_requirements.txt
+
+# install requirements
+RUN curl -sS "https://bootstrap.pypa.io/get-pip.py" | python3.10
+# install known requirements for ComfyUI
+RUN pip install -r /app/ComfyUI/requirements.txt
+# install the detected requirements & ignore errors
+RUN cat /app/docker_requirements.txt | xargs -n 1 -L 1 pip install || true
+# install the extra requirements (manually written)
+RUN pip install -r /app/required_requirements.txt
+
+RUN chmod -R 777 /app
+
+# set environment var
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+
+USER appuser

workflows/comfy/README.md

@@ -0,0 +1,9 @@
+# Docker Setup
+
+1. Run `installer.sh` (this will set up ComfyUI and create a docker_requirements.txt)
+2. Run `docker build -t "storyteller/comfyui" . ` to create the docker container
+
+# Running the container
+
+A folder called `mounts` is where everything will be mounted into the container. Create it and set the
+`COMFY_MOUNTS_DIRECTORY` environment variable in the `storyteller-rs` repo.

workflows/comfy/installer.sh

@@ -0,0 +1,10 @@
+python3 -m venv ComfyUI/venv
+ComfyUI/venv/bin/pip install -r ComfyUI/requirements.txt
+echo "ComfyUI has been installed successfully!"
+ComfyUI/venv/bin/python3 initialize_comfy_extensions.py
+echo "Extensions have been installed successfully!"
+ComfyUI/venv/bin/pip freeze > docker_requirements.txt
+echo "Requirements have been saved to docker_requirements.txt"
+echo "Removing venv..."
+rm -rf ComfyUI/venv
+exit 0

workflows/comfy/required_requirements.txt

@@ -2,3 +2,4 @@ numexpr
 numba
 opencv-python
 color_matcher
+mediapipe

workflows/comfy/run_docker.sh

@@ -0,0 +1,18 @@
+docker run --rm \
+--gpus all \
+--publish 8188:8188 \
+--user $(id -u):$(id -g) \
+-v $(pwd)/mounts/models/checkpoints:/app/ComfyUI/models/checkpoints \
+-v $(pwd)/mounts/models/loras:/app/ComfyUI/models/loras \
+-v $(pwd)/mounts/models/controlnet:/app/ComfyUI/models/controlnet \
+-v $(pwd)/mounts/models/ipadapter:/app/ComfyUI/models/ipadapter \
+-v $(pwd)/mounts/models/clip_vision:/app/ComfyUI/models/clip_vision \
+-v $(pwd)/mounts/prompt:/app/ComfyLauncher/prompt \
+-v $(pwd)/mounts/output:/app/ComfyUI/output \
+-v $(pwd)/mounts/input:/app/ComfyUI/input \
+-v $(pwd)/mounts/custom_nodes/ComfyUI-AnimateDiff-Evolved/models:/app/ComfyUI/custom_nodes/ComfyUI-AnimateDiff-Evolved/models \
+-v $(pwd)/mounts/custom_nodes/comfyui_controlnet_aux/ckpts:/app/ComfyUI/custom_nodes/comfyui_controlnet_aux/ckpts \
+-v $(pwd)/mounts/custom_nodes/ComfyUI-Frame-Interpolation/ckpts:/app/ComfyUI/custom_nodes/ComfyUI-Frame-Interpolation/ckpts \
+-it "storyteller/comfyui" python3 /app/ComfyLauncher/ComfyRunner.py
+
+# -it "test" /bin/bash