From 04e69509cf3de5141cad1756bbcc033bf42b23c8 Mon Sep 17 00:00:00 2001 From: salt Date: Thu, 29 Feb 2024 14:23:56 -0800 Subject: [PATCH] new things --- .../ComfyUI/.ci/update_windows/update.py | 110 ++++++++++++++++++ .../.ci/update_windows/update_comfyui.bat | 8 ++ .../README_VERY_IMPORTANT.txt | 31 +++++ .../.ci/windows_base_files/run_cpu.bat | 2 + .../.ci/windows_base_files/run_nvidia_gpu.bat | 2 + .../ComfyUI/.github/workflows/test-build.yml | 31 +++++ .../ComfyUI/.github/workflows/test-ui.yaml | 26 +++++ .../windows_release_dependencies.yml | 66 +++++++++++ .../windows_release_nightly_pytorch.yml | 90 ++++++++++++++ .../workflows/windows_release_package.yml | 100 ++++++++++++++++ workflows/comfy/ComfyUI/.gitignore | 18 +++ 11 files changed, 484 insertions(+) create mode 100755 workflows/comfy/ComfyUI/.ci/update_windows/update.py create mode 100755 workflows/comfy/ComfyUI/.ci/update_windows/update_comfyui.bat create mode 100755 workflows/comfy/ComfyUI/.ci/windows_base_files/README_VERY_IMPORTANT.txt create mode 100755 workflows/comfy/ComfyUI/.ci/windows_base_files/run_cpu.bat create mode 100755 workflows/comfy/ComfyUI/.ci/windows_base_files/run_nvidia_gpu.bat create mode 100644 workflows/comfy/ComfyUI/.github/workflows/test-build.yml create mode 100644 workflows/comfy/ComfyUI/.github/workflows/test-ui.yaml create mode 100644 workflows/comfy/ComfyUI/.github/workflows/windows_release_dependencies.yml create mode 100644 workflows/comfy/ComfyUI/.github/workflows/windows_release_nightly_pytorch.yml create mode 100644 workflows/comfy/ComfyUI/.github/workflows/windows_release_package.yml create mode 100644 workflows/comfy/ComfyUI/.gitignore diff --git a/workflows/comfy/ComfyUI/.ci/update_windows/update.py b/workflows/comfy/ComfyUI/.ci/update_windows/update.py new file mode 100755 index 0000000..6067d1a --- /dev/null +++ b/workflows/comfy/ComfyUI/.ci/update_windows/update.py @@ -0,0 +1,110 @@ +import pygit2 +from datetime import datetime +import sys +import os +import shutil +import filecmp + +def pull(repo, remote_name='origin', branch='master'): + for remote in repo.remotes: + if remote.name == remote_name: + remote.fetch() + remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target + merge_result, _ = repo.merge_analysis(remote_master_id) + # Up to date, do nothing + if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: + return + # We can just fastforward + elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: + repo.checkout_tree(repo.get(remote_master_id)) + try: + master_ref = repo.lookup_reference('refs/heads/%s' % (branch)) + master_ref.set_target(remote_master_id) + except KeyError: + repo.create_branch(branch, repo.get(remote_master_id)) + repo.head.set_target(remote_master_id) + elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: + repo.merge(remote_master_id) + + if repo.index.conflicts is not None: + for conflict in repo.index.conflicts: + print('Conflicts found in:', conflict[0].path) + raise AssertionError('Conflicts, ahhhhh!!') + + user = repo.default_signature + tree = repo.index.write_tree() + commit = repo.create_commit('HEAD', + user, + user, + 'Merge!', + tree, + [repo.head.target, remote_master_id]) + # We need to do this or git CLI will think we are still merging. + repo.state_cleanup() + else: + raise AssertionError('Unknown merge analysis result') + +pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0) +repo_path = str(sys.argv[1]) +repo = pygit2.Repository(repo_path) +ident = pygit2.Signature('comfyui', 'comfy@ui') +try: + print("stashing current changes") + repo.stash(ident) +except KeyError: + print("nothing to stash") +backup_branch_name = 'backup_branch_{}'.format(datetime.today().strftime('%Y-%m-%d_%H_%M_%S')) +print("creating backup branch: {}".format(backup_branch_name)) +try: + repo.branches.local.create(backup_branch_name, repo.head.peel()) +except: + pass + +print("checking out master branch") +branch = repo.lookup_branch('master') +ref = repo.lookup_reference(branch.name) +repo.checkout(ref) + +print("pulling latest changes") +pull(repo) + +print("Done!") + +self_update = True +if len(sys.argv) > 2: + self_update = '--skip_self_update' not in sys.argv + +update_py_path = os.path.realpath(__file__) +repo_update_py_path = os.path.join(repo_path, ".ci/update_windows/update.py") + +cur_path = os.path.dirname(update_py_path) + + +req_path = os.path.join(cur_path, "current_requirements.txt") +repo_req_path = os.path.join(repo_path, "requirements.txt") + + +def files_equal(file1, file2): + try: + return filecmp.cmp(file1, file2, shallow=False) + except: + return False + +def file_size(f): + try: + return os.path.getsize(f) + except: + return 0 + + +if self_update and not files_equal(update_py_path, repo_update_py_path) and file_size(repo_update_py_path) > 10: + shutil.copy(repo_update_py_path, os.path.join(cur_path, "update_new.py")) + exit() + +if not os.path.exists(req_path) or not files_equal(repo_req_path, req_path): + import subprocess + try: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-r', repo_req_path]) + shutil.copy(repo_req_path, req_path) + except: + pass diff --git a/workflows/comfy/ComfyUI/.ci/update_windows/update_comfyui.bat b/workflows/comfy/ComfyUI/.ci/update_windows/update_comfyui.bat new file mode 100755 index 0000000..bb08c0d --- /dev/null +++ b/workflows/comfy/ComfyUI/.ci/update_windows/update_comfyui.bat @@ -0,0 +1,8 @@ +@echo off +..\python_embeded\python.exe .\update.py ..\ComfyUI\ +if exist update_new.py ( + move /y update_new.py update.py + echo Running updater again since it got updated. + ..\python_embeded\python.exe .\update.py ..\ComfyUI\ --skip_self_update +) +if "%~1"=="" pause diff --git a/workflows/comfy/ComfyUI/.ci/windows_base_files/README_VERY_IMPORTANT.txt b/workflows/comfy/ComfyUI/.ci/windows_base_files/README_VERY_IMPORTANT.txt new file mode 100755 index 0000000..0216658 --- /dev/null +++ b/workflows/comfy/ComfyUI/.ci/windows_base_files/README_VERY_IMPORTANT.txt @@ -0,0 +1,31 @@ +HOW TO RUN: + +if you have a NVIDIA gpu: + +run_nvidia_gpu.bat + + + +To run it in slow CPU mode: + +run_cpu.bat + + + +IF YOU GET A RED ERROR IN THE UI MAKE SURE YOU HAVE A MODEL/CHECKPOINT IN: ComfyUI\models\checkpoints + +You can download the stable diffusion 1.5 one from: https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.ckpt + + +RECOMMENDED WAY TO UPDATE: +To update the ComfyUI code: update\update_comfyui.bat + + + +To update ComfyUI with the python dependencies, note that you should ONLY run this if you have issues with python dependencies. +update\update_comfyui_and_python_dependencies.bat + + +TO SHARE MODELS BETWEEN COMFYUI AND ANOTHER UI: +In the ComfyUI directory you will find a file: extra_model_paths.yaml.example +Rename this file to: extra_model_paths.yaml and edit it with your favorite text editor. diff --git a/workflows/comfy/ComfyUI/.ci/windows_base_files/run_cpu.bat b/workflows/comfy/ComfyUI/.ci/windows_base_files/run_cpu.bat new file mode 100755 index 0000000..c3ba417 --- /dev/null +++ b/workflows/comfy/ComfyUI/.ci/windows_base_files/run_cpu.bat @@ -0,0 +1,2 @@ +.\python_embeded\python.exe -s ComfyUI\main.py --cpu --windows-standalone-build +pause diff --git a/workflows/comfy/ComfyUI/.ci/windows_base_files/run_nvidia_gpu.bat b/workflows/comfy/ComfyUI/.ci/windows_base_files/run_nvidia_gpu.bat new file mode 100755 index 0000000..274d7c9 --- /dev/null +++ b/workflows/comfy/ComfyUI/.ci/windows_base_files/run_nvidia_gpu.bat @@ -0,0 +1,2 @@ +.\python_embeded\python.exe -s ComfyUI\main.py --windows-standalone-build +pause diff --git a/workflows/comfy/ComfyUI/.github/workflows/test-build.yml b/workflows/comfy/ComfyUI/.github/workflows/test-build.yml new file mode 100644 index 0000000..444d6b2 --- /dev/null +++ b/workflows/comfy/ComfyUI/.github/workflows/test-build.yml @@ -0,0 +1,31 @@ +name: Build package + +# +# This workflow is a test of the python package build. +# Install Python dependencies across different Python versions. +# + +on: + push: + paths: + - "requirements.txt" + - ".github/workflows/test-build.yml" + +jobs: + build: + name: Build Test + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt \ No newline at end of file diff --git a/workflows/comfy/ComfyUI/.github/workflows/test-ui.yaml b/workflows/comfy/ComfyUI/.github/workflows/test-ui.yaml new file mode 100644 index 0000000..4b8b979 --- /dev/null +++ b/workflows/comfy/ComfyUI/.github/workflows/test-ui.yaml @@ -0,0 +1,26 @@ +name: Tests CI + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install requirements + run: | + python -m pip install --upgrade pip + pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + pip install -r requirements.txt + - name: Run Tests + run: | + npm ci + npm run test:generate + npm test -- --verbose + working-directory: ./tests-ui diff --git a/workflows/comfy/ComfyUI/.github/workflows/windows_release_dependencies.yml b/workflows/comfy/ComfyUI/.github/workflows/windows_release_dependencies.yml new file mode 100644 index 0000000..e0841fd --- /dev/null +++ b/workflows/comfy/ComfyUI/.github/workflows/windows_release_dependencies.yml @@ -0,0 +1,66 @@ +name: "Windows Release dependencies" + +on: + workflow_dispatch: + inputs: + xformers: + description: 'xformers version' + required: false + type: string + default: "" + cu: + description: 'cuda version' + required: true + type: string + default: "121" + + python_minor: + description: 'python minor version' + required: true + type: string + default: "11" + + python_patch: + description: 'python patch version' + required: true + type: string + default: "6" +# push: +# branches: +# - master + +jobs: + build_dependencies: + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.${{ inputs.python_minor }}.${{ inputs.python_patch }} + + - shell: bash + run: | + echo "@echo off + call update_comfyui.bat nopause + echo - + echo This will try to update pytorch and all python dependencies. + echo - + echo If you just want to update normally, close this and run update_comfyui.bat instead. + echo - + pause + ..\python_embeded\python.exe -s -m pip install --upgrade torch torchvision torchaudio ${{ inputs.xformers }} --extra-index-url https://download.pytorch.org/whl/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 + pause" > update_comfyui_and_python_dependencies.bat + + python -m pip wheel --no-cache-dir torch torchvision torchaudio ${{ inputs.xformers }} --extra-index-url https://download.pytorch.org/whl/cu${{ inputs.cu }} -r requirements.txt pygit2 -w ./temp_wheel_dir + python -m pip install --no-cache-dir ./temp_wheel_dir/* + echo installed basic + ls -lah temp_wheel_dir + mv temp_wheel_dir cu${{ inputs.cu }}_python_deps + tar cf cu${{ inputs.cu }}_python_deps.tar cu${{ inputs.cu }}_python_deps + + - uses: actions/cache/save@v3 + with: + path: | + cu${{ inputs.cu }}_python_deps.tar + update_comfyui_and_python_dependencies.bat + key: ${{ runner.os }}-build-cu${{ inputs.cu }}-${{ inputs.python_minor }} diff --git a/workflows/comfy/ComfyUI/.github/workflows/windows_release_nightly_pytorch.yml b/workflows/comfy/ComfyUI/.github/workflows/windows_release_nightly_pytorch.yml new file mode 100644 index 0000000..5683093 --- /dev/null +++ b/workflows/comfy/ComfyUI/.github/workflows/windows_release_nightly_pytorch.yml @@ -0,0 +1,90 @@ +name: "Windows Release Nightly pytorch" + +on: + workflow_dispatch: + inputs: + cu: + description: 'cuda version' + required: true + type: string + default: "121" + + python_minor: + description: 'python minor version' + required: true + type: string + default: "12" + + python_patch: + description: 'python patch version' + required: true + type: string + default: "1" +# push: +# branches: +# - master + +jobs: + build: + permissions: + contents: "write" + packages: "write" + pull-requests: "read" + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v4 + with: + python-version: 3.${{ inputs.python_minor }}.${{ inputs.python_patch }} + - shell: bash + run: | + cd .. + cp -r ComfyUI ComfyUI_copy + curl https://www.python.org/ftp/python/3.${{ inputs.python_minor }}.${{ inputs.python_patch }}/python-3.${{ inputs.python_minor }}.${{ inputs.python_patch }}-embed-amd64.zip -o python_embeded.zip + unzip python_embeded.zip -d python_embeded + cd python_embeded + echo 'import site' >> ./python3${{ inputs.python_minor }}._pth + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + ./python.exe get-pip.py + python -m pip wheel torch torchvision torchaudio --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir + ls ../temp_wheel_dir + ./python.exe -s -m pip install --pre ../temp_wheel_dir/* + sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth + cd .. + + git clone https://github.com/comfyanonymous/taesd + cp taesd/*.pth ./ComfyUI_copy/models/vae_approx/ + + mkdir ComfyUI_windows_portable_nightly_pytorch + mv python_embeded ComfyUI_windows_portable_nightly_pytorch + mv ComfyUI_copy ComfyUI_windows_portable_nightly_pytorch/ComfyUI + + cd ComfyUI_windows_portable_nightly_pytorch + + mkdir update + cp -r ComfyUI/.ci/update_windows/* ./update/ + cp -r ComfyUI/.ci/windows_base_files/* ./ + + echo "call update_comfyui.bat nopause + ..\python_embeded\python.exe -s -m pip install --upgrade --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 + pause" > ./update/update_comfyui_and_python_dependencies.bat + cd .. + + "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on -mf=BCJ2 ComfyUI_windows_portable_nightly_pytorch.7z ComfyUI_windows_portable_nightly_pytorch + mv ComfyUI_windows_portable_nightly_pytorch.7z ComfyUI/ComfyUI_windows_portable_nvidia_or_cpu_nightly_pytorch.7z + + cd ComfyUI_windows_portable_nightly_pytorch + python_embeded/python.exe -s ComfyUI/main.py --quick-test-for-ci --cpu + + ls + + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ComfyUI_windows_portable_nvidia_or_cpu_nightly_pytorch.7z + tag: "latest" + overwrite: true diff --git a/workflows/comfy/ComfyUI/.github/workflows/windows_release_package.yml b/workflows/comfy/ComfyUI/.github/workflows/windows_release_package.yml new file mode 100644 index 0000000..87d37c2 --- /dev/null +++ b/workflows/comfy/ComfyUI/.github/workflows/windows_release_package.yml @@ -0,0 +1,100 @@ +name: "Windows Release packaging" + +on: + workflow_dispatch: + inputs: + cu: + description: 'cuda version' + required: true + type: string + default: "121" + + python_minor: + description: 'python minor version' + required: true + type: string + default: "11" + + python_patch: + description: 'python patch version' + required: true + type: string + default: "6" +# push: +# branches: +# - master + +jobs: + package_comfyui: + permissions: + contents: "write" + packages: "write" + pull-requests: "read" + runs-on: windows-latest + steps: + - uses: actions/cache/restore@v3 + id: cache + with: + path: | + cu${{ inputs.cu }}_python_deps.tar + update_comfyui_and_python_dependencies.bat + key: ${{ runner.os }}-build-cu${{ inputs.cu }}-${{ inputs.python_minor }} + - shell: bash + run: | + mv cu${{ inputs.cu }}_python_deps.tar ../ + mv update_comfyui_and_python_dependencies.bat ../ + cd .. + tar xf cu${{ inputs.cu }}_python_deps.tar + pwd + ls + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + persist-credentials: false + - shell: bash + run: | + cd .. + cp -r ComfyUI ComfyUI_copy + curl https://www.python.org/ftp/python/3.${{ inputs.python_minor }}.${{ inputs.python_patch }}/python-3.${{ inputs.python_minor }}.${{ inputs.python_patch }}-embed-amd64.zip -o python_embeded.zip + unzip python_embeded.zip -d python_embeded + cd python_embeded + echo 'import site' >> ./python3${{ inputs.python_minor }}._pth + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + ./python.exe get-pip.py + ./python.exe -s -m pip install ../cu${{ inputs.cu }}_python_deps/* + sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth + cd .. + + git clone https://github.com/comfyanonymous/taesd + cp taesd/*.pth ./ComfyUI_copy/models/vae_approx/ + + mkdir ComfyUI_windows_portable + mv python_embeded ComfyUI_windows_portable + mv ComfyUI_copy ComfyUI_windows_portable/ComfyUI + + cd ComfyUI_windows_portable + + mkdir update + cp -r ComfyUI/.ci/update_windows/* ./update/ + cp -r ComfyUI/.ci/windows_base_files/* ./ + cp ../update_comfyui_and_python_dependencies.bat ./update/ + + cd .. + + "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable + mv ComfyUI_windows_portable.7z ComfyUI/new_ComfyUI_windows_portable_nvidia_cu${{ inputs.cu }}_or_cpu.7z + + cd ComfyUI_windows_portable + python_embeded/python.exe -s ComfyUI/main.py --quick-test-for-ci --cpu + + ls + + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: new_ComfyUI_windows_portable_nvidia_cu${{ inputs.cu }}_or_cpu.7z + tag: "latest" + overwrite: true + diff --git a/workflows/comfy/ComfyUI/.gitignore b/workflows/comfy/ComfyUI/.gitignore new file mode 100644 index 0000000..9f03892 --- /dev/null +++ b/workflows/comfy/ComfyUI/.gitignore @@ -0,0 +1,18 @@ +__pycache__/ +*.py[cod] +/output/ +/input/ +!/input/example.png +/models/ +/temp/ +/custom_nodes/ +!custom_nodes/example_node.py.example +extra_model_paths.yaml +/.vs +.idea/ +venv/ +/web/extensions/* +!/web/extensions/logging.js.example +!/web/extensions/core/ +/tests-ui/data/object_info.json +/user/ \ No newline at end of file