build comfy w/ more layers

workflows/comfy/Dockerfile

@@ -36,18 +36,32 @@ RUN apt-get update \
         --no-install-recommends
 
 RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser
-# Copy files to container
-COPY --chown=appuser:appgroup initialize_comfy_extensions.py /app/initialize_comfy_extensions.py
-COPY --chown=appuser:appgroup install.py /app/install.py
-COPY --chown=appuser:appgroup repos.json /app/repos.json
-COPY --chown=appuser:appgroup ComfyLauncher /app/ComfyLauncher
-COPY --chown=appuser:appgroup format_video.py /app/format_video.py
-COPY --chown=appuser:appgroup required_requirements.txt /app/required_requirements.txt
-COPY --chown=appuser:appgroup dependencies.json /app/dependencies.json
 
 RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel requests argparse
 
 RUN chmod -R 777 /app
+WORKDIR /app
+
+# Copy files to container
+COPY --chown=appuser:appgroup initialize_comfy_extensions.py /app/initialize_comfy_extensions.py
+COPY --chown=appuser:appgroup ComfyLauncher /app/ComfyLauncher
+COPY --chown=appuser:appgroup format_video.py /app/format_video.py
+COPY --chown=appuser:appgroup install.py /app/install.py
+
+# download & install comfy ui and its dependencies
+RUN python3 install.py --comfy-ui-url https://github.com/comfyanonymous/ComfyUI.git --comfy-ui-commit eda87043862f743b0a0467735f8531f7c4709b3a
+RUN python3 -m venv ComfyUI/venv
+RUN ComfyUI/venv/bin/python -m pip install -r ComfyUI/requirements.txt
+
+
+COPY --chown=appuser:appgroup required_requirements.txt /app/required_requirements.txt
+RUN ComfyUI/venv/bin/python -m pip install -r required_requirements.txt
+
+
+COPY --chown=appuser:appgroup repos.json /app/repos.json
+RUN python3 install.py --repos-json repos.json
+
+COPY --chown=appuser:appgroup dependencies.json /app/dependencies.json
 
 # protobuf use python implementation (weird error without)
 ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python

workflows/comfy/install.py

@@ -3,12 +3,27 @@ import subprocess
 import json
 import os
 from pathlib import Path
-import requests
+from urllib.parse import urlparse
+from multiprocessing import Pool
+import argparse
+
+parser = argparse.ArgumentParser(description='Install ComfyUI and its dependencies')
+parser.add_argument('--source', type=str, help='The source directory of the dependencies', required=False)
+parser.add_argument('--dependencies-json', type=str, help='The path to the json file containing the dependencies', required=False)
+parser.add_argument('--repos-json', type=str, help='The path to the json file containing the repositories', required=False)
+
+parser.add_argument('--concurrency', type=int, help='The number of concurrent processes to use', default=8)
+
+parser.add_argument('--comfy-ui-url', type=str, help='The url of the ComfyUI repository', required=False)
+parser.add_argument('--comfy-ui-commit', type=str, help='The commit of the ComfyUI repository', required=False)
+
+
+args = parser.parse_args()
+
 
 # set cwd to script location
 os.chdir(os.path.dirname(os.path.realpath(__file__)))
 
-json_file_path = 'repos.json'
 venv_python = os.path.join('ComfyUI', 'venv', 'Scripts' if os.name == 'nt' else 'bin', 'python')
 
 
@@ -29,7 +44,6 @@ def run_command(command):
             print(line, end='')
         print(f"Error executing command: {command}")
 
-
 def setup_repo(repo_info):
     url, commit, folder = repo_info['url'], repo_info['commit'], repo_info['folder']
     # save the current working directory
@@ -42,6 +56,15 @@ def setup_repo(repo_info):
         os.chdir(original_cwd)
 
 
+def setup_repos(json_file_path, concurrency=4):
+    with open(json_file_path, 'r') as file:
+        repos = json.load(file)
+    with Pool(concurrency) as pool:
+        pool.map(setup_repo, repos)
+
+def setup_comfy_ui(comfyui_info):
+    setup_repo(comfyui_info)
+
 def setup_venv_and_install_requirements():
     run_command("python3 -m venv ComfyUI/venv")
     print('=============# Installing ComfyUI Requirements #=============')
@@ -50,50 +73,53 @@ def setup_venv_and_install_requirements():
     run_command(f"{venv_python} -m pip install -r required_requirements.txt")
 
 
-def run_comfyui_setup():
+def initialize_comfyui():
     run_command(f"{venv_python} initialize_comfy_extensions.py")
 
+def install_dependency(dep, source_dir):
+    location = dep['location']
+    url = dep['url']
 
-def install_dependencies(json_file_path='dependencies.json'):
+    # separate the pathname from the url host
+    source_pathname = urlparse(url).path.lstrip('/')
+
+    # Convert the location to a Path object and create parent directories if they don't exist
+    path = Path(location)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    source_path = Path() / source_dir /  source_pathname
+    # Copy the dependency from source dir to the location
+    print(f"Copying checkpoint from {source_path} to {location}")
+    run_command(f"cp {source_path} {location}")
+
+def install_dependencies(json_file_path, source_dir, concurrency=4):
     # Load the dependency JSON
     with open(json_file_path, 'r') as file:
         dependencies = json.load(file)
 
-    for dep in dependencies:
-        # Extract location and URL
-        location = dep['location']
-        url = dep['url']
-
-        # Convert the location to a Path object and create parent directories if they don't exist
-        path = Path(location)
-        path.parent.mkdir(parents=True, exist_ok=True)
-
-        if path.is_file():
-            print(f"{path} is already installed")
-            continue
-
-        # Download the file
-        response = requests.get(url)
-        if response.status_code == 200:
-            # Write the content to the file in the specified location
-            with open(path, 'wb') as file:
-                file.write(response.content)
-            print(f"Installed {path}")
-        else:
-            print(f"Failed to download {url}")
-
+    with Pool(concurrency) as pool:
+        pool.starmap(install_dependency, [(dep, source_dir) for dep in dependencies])
 
 def main():
-    with open(json_file_path, 'r') as file:
-        repos = json.load(file)
+    if args.comfy_ui_url and args.comfy_ui_commit:
+        comfyui_info = {
+            "url": args.comfy_ui_url,
+            "commit": args.comfy_ui_commit,
+            "folder": "ComfyUI",
+        }
+        setup_comfy_ui(comfyui_info)
 
-    for repo_info in repos:
-        print(f"installing {repo_info['url']} at commit {repo_info['commit']} to {repo_info['folder']}")
-        setup_repo(repo_info)
+    if args.repos_json:
+        setup_repos(args.repos_json, args.concurrency)
+    else:
+        print("Skipping repository setup as repos json is not provided")
 
-    setup_venv_and_install_requirements()
-    install_dependencies()
-    run_comfyui_setup()
+
+    initialize_comfyui()
+    if args.source and args.dependencies_json:
+        install_dependencies(args.dependencies_json, args.source, args.concurrency)
+    else:
+        print("Skipping dependency installation as source and dependencies json are not provided")
 
 
 if __name__ == "__main__":

workflows/comfy/repos.json

@@ -1,9 +1,4 @@
 [
-    {
-        "url": "https://github.com/comfyanonymous/ComfyUI.git",
-        "commit": "eda87043862f743b0a0467735f8531f7c4709b3a",
-        "folder": "ComfyUI"
-    },
     {
         "url": "https://github.com/ltdrdata/ComfyUI-Manager.git",
         "commit": "6c553d1228978352767f4914c4dd7787b00f6769",