From 9031a6be25d9b6112ea560e6a15fb51e8143b959 Mon Sep 17 00:00:00 2001 From: Kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Mon, 18 Mar 2024 17:21:13 +0530 Subject: [PATCH] build comfy w/ more layers --- workflows/comfy/Dockerfile | 30 ++++++++---- workflows/comfy/install.py | 98 ++++++++++++++++++++++++-------------- workflows/comfy/repos.json | 5 -- 3 files changed, 84 insertions(+), 49 deletions(-) diff --git a/workflows/comfy/Dockerfile b/workflows/comfy/Dockerfile index ef6ddb8..48f2e5d 100644 --- a/workflows/comfy/Dockerfile +++ b/workflows/comfy/Dockerfile @@ -36,18 +36,32 @@ RUN apt-get update \ --no-install-recommends RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser -# Copy files to container -COPY --chown=appuser:appgroup initialize_comfy_extensions.py /app/initialize_comfy_extensions.py -COPY --chown=appuser:appgroup install.py /app/install.py -COPY --chown=appuser:appgroup repos.json /app/repos.json -COPY --chown=appuser:appgroup ComfyLauncher /app/ComfyLauncher -COPY --chown=appuser:appgroup format_video.py /app/format_video.py -COPY --chown=appuser:appgroup required_requirements.txt /app/required_requirements.txt -COPY --chown=appuser:appgroup dependencies.json /app/dependencies.json RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel requests argparse RUN chmod -R 777 /app +WORKDIR /app + +# Copy files to container +COPY --chown=appuser:appgroup initialize_comfy_extensions.py /app/initialize_comfy_extensions.py +COPY --chown=appuser:appgroup ComfyLauncher /app/ComfyLauncher +COPY --chown=appuser:appgroup format_video.py /app/format_video.py +COPY --chown=appuser:appgroup install.py /app/install.py + +# download & install comfy ui and its dependencies +RUN python3 install.py --comfy-ui-url https://github.com/comfyanonymous/ComfyUI.git --comfy-ui-commit eda87043862f743b0a0467735f8531f7c4709b3a +RUN python3 -m venv ComfyUI/venv +RUN ComfyUI/venv/bin/python -m pip install -r ComfyUI/requirements.txt + + +COPY --chown=appuser:appgroup required_requirements.txt /app/required_requirements.txt +RUN ComfyUI/venv/bin/python -m pip install -r required_requirements.txt + + +COPY --chown=appuser:appgroup repos.json /app/repos.json +RUN python3 install.py --repos-json repos.json + +COPY --chown=appuser:appgroup dependencies.json /app/dependencies.json # protobuf use python implementation (weird error without) ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python diff --git a/workflows/comfy/install.py b/workflows/comfy/install.py index d83139f..f7c20bb 100644 --- a/workflows/comfy/install.py +++ b/workflows/comfy/install.py @@ -3,12 +3,27 @@ import subprocess import json import os from pathlib import Path -import requests +from urllib.parse import urlparse +from multiprocessing import Pool +import argparse + +parser = argparse.ArgumentParser(description='Install ComfyUI and its dependencies') +parser.add_argument('--source', type=str, help='The source directory of the dependencies', required=False) +parser.add_argument('--dependencies-json', type=str, help='The path to the json file containing the dependencies', required=False) +parser.add_argument('--repos-json', type=str, help='The path to the json file containing the repositories', required=False) + +parser.add_argument('--concurrency', type=int, help='The number of concurrent processes to use', default=8) + +parser.add_argument('--comfy-ui-url', type=str, help='The url of the ComfyUI repository', required=False) +parser.add_argument('--comfy-ui-commit', type=str, help='The commit of the ComfyUI repository', required=False) + + +args = parser.parse_args() + # set cwd to script location os.chdir(os.path.dirname(os.path.realpath(__file__))) -json_file_path = 'repos.json' venv_python = os.path.join('ComfyUI', 'venv', 'Scripts' if os.name == 'nt' else 'bin', 'python') @@ -29,7 +44,6 @@ def run_command(command): print(line, end='') print(f"Error executing command: {command}") - def setup_repo(repo_info): url, commit, folder = repo_info['url'], repo_info['commit'], repo_info['folder'] # save the current working directory @@ -42,6 +56,15 @@ def setup_repo(repo_info): os.chdir(original_cwd) +def setup_repos(json_file_path, concurrency=4): + with open(json_file_path, 'r') as file: + repos = json.load(file) + with Pool(concurrency) as pool: + pool.map(setup_repo, repos) + +def setup_comfy_ui(comfyui_info): + setup_repo(comfyui_info) + def setup_venv_and_install_requirements(): run_command("python3 -m venv ComfyUI/venv") print('=============# Installing ComfyUI Requirements #=============') @@ -50,50 +73,53 @@ def setup_venv_and_install_requirements(): run_command(f"{venv_python} -m pip install -r required_requirements.txt") -def run_comfyui_setup(): +def initialize_comfyui(): run_command(f"{venv_python} initialize_comfy_extensions.py") +def install_dependency(dep, source_dir): + location = dep['location'] + url = dep['url'] -def install_dependencies(json_file_path='dependencies.json'): + # separate the pathname from the url host + source_pathname = urlparse(url).path.lstrip('/') + + # Convert the location to a Path object and create parent directories if they don't exist + path = Path(location) + path.parent.mkdir(parents=True, exist_ok=True) + + source_path = Path() / source_dir / source_pathname + # Copy the dependency from source dir to the location + print(f"Copying checkpoint from {source_path} to {location}") + run_command(f"cp {source_path} {location}") + +def install_dependencies(json_file_path, source_dir, concurrency=4): # Load the dependency JSON with open(json_file_path, 'r') as file: dependencies = json.load(file) - for dep in dependencies: - # Extract location and URL - location = dep['location'] - url = dep['url'] - - # Convert the location to a Path object and create parent directories if they don't exist - path = Path(location) - path.parent.mkdir(parents=True, exist_ok=True) - - if path.is_file(): - print(f"{path} is already installed") - continue - - # Download the file - response = requests.get(url) - if response.status_code == 200: - # Write the content to the file in the specified location - with open(path, 'wb') as file: - file.write(response.content) - print(f"Installed {path}") - else: - print(f"Failed to download {url}") - + with Pool(concurrency) as pool: + pool.starmap(install_dependency, [(dep, source_dir) for dep in dependencies]) def main(): - with open(json_file_path, 'r') as file: - repos = json.load(file) + if args.comfy_ui_url and args.comfy_ui_commit: + comfyui_info = { + "url": args.comfy_ui_url, + "commit": args.comfy_ui_commit, + "folder": "ComfyUI", + } + setup_comfy_ui(comfyui_info) - for repo_info in repos: - print(f"installing {repo_info['url']} at commit {repo_info['commit']} to {repo_info['folder']}") - setup_repo(repo_info) + if args.repos_json: + setup_repos(args.repos_json, args.concurrency) + else: + print("Skipping repository setup as repos json is not provided") - setup_venv_and_install_requirements() - install_dependencies() - run_comfyui_setup() + + initialize_comfyui() + if args.source and args.dependencies_json: + install_dependencies(args.dependencies_json, args.source, args.concurrency) + else: + print("Skipping dependency installation as source and dependencies json are not provided") if __name__ == "__main__": diff --git a/workflows/comfy/repos.json b/workflows/comfy/repos.json index d8cf56b..9ebb5ca 100644 --- a/workflows/comfy/repos.json +++ b/workflows/comfy/repos.json @@ -1,9 +1,4 @@ [ - { - "url": "https://github.com/comfyanonymous/ComfyUI.git", - "commit": "eda87043862f743b0a0467735f8531f7c4709b3a", - "folder": "ComfyUI" - }, { "url": "https://github.com/ltdrdata/ComfyUI-Manager.git", "commit": "6c553d1228978352767f4914c4dd7787b00f6769",