more extensions

2026-10-12 23:48:57 +00:00 · 2024-02-20 08:30:59 -08:00
parent 8048a05d12
commit 7053eec79c
145 changed files with 7747 additions and 2775 deletions
@@ -1,5 +1,5 @@
 # ComfyUI-Advanced-ControlNet
-Nodes for scheduling ControlNet strength across timesteps and batched latents, as well as applying custom weights and attention masks. The ControlNet nodes here fully support sliding context sampling, like the one used in the  [ComfyUI-AnimateDiff-Evolved](https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved) nodes. Currently supports ControlNets, T2IAdapters, ControlLoRAs, ControlLLLite, and SparseCtrls.
+Nodes for scheduling ControlNet strength across timesteps and batched latents, as well as applying custom weights and attention masks. The ControlNet nodes here fully support sliding context sampling, like the one used in the  [ComfyUI-AnimateDiff-Evolved](https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved) nodes. Currently supports ControlNets, T2IAdapters, ControlLoRAs, ControlLLLite, SparseCtrls, and SVD-ControlNets.

 Custom weights allow replication of the "My prompt is more important" feature of Auto1111's sd-webui ControlNet extension.

@@ -12,6 +12,8 @@ ControlNet preprocessors are available through [comfyui_controlnet_aux](https://
 - ControlNet, T2IAdapter, and ControlLoRA support for sliding context windows.
 - ControlLLLite support (requires model_optional to be passed into and out of Apply Advanced ControlNet node)
 - SparseCtrl support
+- SVD-ControlNet support
+  - Stable Video Diffusion ControlNets trained by **CiaraRowles**: [Depth](https://huggingface.co/CiaraRowles/temporal-controlnet-depth-svd-v1/tree/main/controlnet), [Lineart](https://huggingface.co/CiaraRowles/temporal-controlnet-lineart-svd-v1/tree/main/controlnet)  

 ## Table of Contents:
 - [Scheduling Explanation](#scheduling-explanation)
@@ -12,6 +12,7 @@ from model_patcher import ModelPatcher

 from .control_sparsectrl import SparseControlNet, SparseCtrlMotionWrapper, SparseMethod, SparseSettings, SparseSpreadMethod, PreprocSparseRGBWrapper
 from .control_lllite import LLLiteModule, LLLitePatch
+from .control_svd import svd_unet_config_from_diffusers_unet, SVDControlNet, svd_unet_to_diffusers
 from .utils import (AdvancedControlBase, TimestepKeyframeGroup, LatentKeyframeGroup, ControlWeightType, ControlWeights, WeightTypeException,
                    manual_cast_clean_groupnorm, disable_weight_init_clean_groupnorm, prepare_mask_batch, get_properly_arranged_t2i_weights, load_torch_file_with_dict_factory)
 from .logger import logger
@@ -64,7 +65,7 @@ class ControlNetAdvanced(ControlNet, AdvancedControlBase):
        # prepare mask_cond_hint
        self.prepare_mask_cond_hint(x_noisy=x_noisy, t=t, cond=cond, batched_number=batched_number, dtype=dtype)

-        context = cond['c_crossattn']
+        context = cond.get('crossattn_controlnet', cond['c_crossattn'])
        # uses 'y' in new ComfyUI update
        y = cond.get('y', None)
        if y is None: # TODO: remove this in the future since no longer used by newest ComfyUI
@@ -168,6 +169,71 @@ class ControlLoraAdvanced(ControlLora, AdvancedControlBase):
                                   global_average_pooling=v.global_average_pooling, device=v.device)


+class SVDControlNetAdvanced(ControlNetAdvanced):
+    def __init__(self, control_model: SVDControlNet, timestep_keyframes: TimestepKeyframeGroup, global_average_pooling=False, device=None, load_device=None, manual_cast_dtype=None):
+        super().__init__(control_model=control_model, timestep_keyframes=timestep_keyframes, global_average_pooling=global_average_pooling, device=device, load_device=load_device, manual_cast_dtype=manual_cast_dtype)
+
+    def set_cond_hint(self, *args, **kwargs):
+        to_return = super().set_cond_hint(*args, **kwargs)
+        # cond hint for SVD-ControlNet needs to be scaled between (-1, 1) instead of (0, 1)
+        self.cond_hint_original = self.cond_hint_original * 2.0 - 1.0
+        return to_return
+
+    def get_control_advanced(self, x_noisy, t, cond, batched_number):
+        control_prev = None
+        if self.previous_controlnet is not None:
+            control_prev = self.previous_controlnet.get_control(x_noisy, t, cond, batched_number)
+
+        if self.timestep_range is not None:
+            if t[0] > self.timestep_range[0] or t[0] < self.timestep_range[1]:
+                if control_prev is not None:
+                    return control_prev
+                else:
+                    return None
+
+        dtype = self.control_model.dtype
+        if self.manual_cast_dtype is not None:
+            dtype = self.manual_cast_dtype
+
+        output_dtype = x_noisy.dtype
+        # make cond_hint appropriate dimensions
+        # TODO: change this to not require cond_hint upscaling every step when self.sub_idxs are present
+        if self.sub_idxs is not None or self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
+            if self.cond_hint is not None:
+                del self.cond_hint
+            self.cond_hint = None
+            # if self.cond_hint_original length greater or equal to real latent count, subdivide it before scaling
+            if self.sub_idxs is not None and self.cond_hint_original.size(0) >= self.full_latent_length:
+                self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original[self.sub_idxs], x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(dtype).to(self.device)
+            else:
+                self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(dtype).to(self.device)
+        if x_noisy.shape[0] != self.cond_hint.shape[0]:
+            self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number)
+
+        # prepare mask_cond_hint
+        self.prepare_mask_cond_hint(x_noisy=x_noisy, t=t, cond=cond, batched_number=batched_number, dtype=dtype)
+
+        context = cond.get('crossattn_controlnet', cond['c_crossattn'])
+        # uses 'y' in new ComfyUI update
+        y = cond.get('y', None)
+        if y is not None:
+            y = y.to(dtype)
+        timestep = self.model_sampling_current.timestep(t)
+        x_noisy = self.model_sampling_current.calculate_input(t, x_noisy)
+        # concat c_concat if exists (should exist for SVD), doubling channels to 8
+        if cond.get('c_concat', None) is not None:
+            x_noisy = torch.cat([x_noisy] + [cond['c_concat']], dim=1)
+
+        control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(dtype), y=y, cond=cond)
+        return self.control_merge(None, control, control_prev, output_dtype)
+
+    def copy(self):
+        c = SVDControlNetAdvanced(self.control_model, self.timestep_keyframes, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype)
+        self.copy_to(c)
+        self.copy_to_advanced(c)
+        return c
+
+
 class SparseCtrlAdvanced(ControlNetAdvanced):
    def __init__(self, control_model, timestep_keyframes: TimestepKeyframeGroup, sparse_settings: SparseSettings=None, global_average_pooling=False, device=None, load_device=None, manual_cast_dtype=None):
        super().__init__(control_model=control_model, timestep_keyframes=timestep_keyframes, global_average_pooling=global_average_pooling, device=device, load_device=load_device, manual_cast_dtype=manual_cast_dtype)
@@ -282,27 +348,19 @@ class SparseCtrlAdvanced(ControlNetAdvanced):
        return c


-class ControlLLLiteAdvanced(ControlBase, AdvancedControlBase):
-    # This ControlNet is more of an attention patch than a traditional controlnet
-    def __init__(self, patch: LLLitePatch, timestep_keyframes: TimestepKeyframeGroup, device=None):
+class ReferenceAdvanced(ControlBase, AdvancedControlBase):
+    def __init__(self, timestep_keyframes: TimestepKeyframeGroup, device=None):
        super().__init__(device)
        AdvancedControlBase.__init__(self, super(), timestep_keyframes=timestep_keyframes, weights_default=ControlWeights.controllllite(), require_model=True)
-        self.patch = patch.clone_with_control(self)
-
+        # TODO: save attn patches here
+    
    def patch_model(self, model: ModelPatcher):
-        model.set_model_attn1_patch(self.patch)
-        model.set_model_attn2_patch(self.patch)
-
-    def set_cond_hint(self, *args, **kwargs):
-        to_return = super().set_cond_hint(*args, **kwargs)
-        # cond hint for LLLite needs to be scaled between (-1, 1) instead of (0, 1)
-        self.cond_hint_original = self.cond_hint_original * 2.0 - 1.0
-        return to_return
+        # TODO: do model patching here
+        pass

    def pre_run_advanced(self, *args, **kwargs):
        AdvancedControlBase.pre_run_advanced(self, *args, **kwargs)
-        self.patch.set_control(self)
-        #logger.warn(f"in pre_run_advanced: {id(self)}")
+        # TODO: set control on patches
    
    def get_control_advanced(self, x_noisy: Tensor, t, cond, batched_number: int):
        # normal ControlNet stuff
@@ -332,24 +390,121 @@ class ControlLLLiteAdvanced(ControlBase, AdvancedControlBase):
        # done preparing; model patches will take care of everything now.
        # return normal controlnet stuff
        return control_prev
+
+    def cleanup_advanced(self):
+        super().cleanup_advanced()
+        # TODO: cleanup patches here
+    
+    def copy(self):
+        c = ReferenceAdvanced(self.timestep_keyframes)
+        self.copy_to(c)
+        self.copy_to_advanced(c)
+        return c
+
+
+class ControlLLLiteAdvanced(ControlBase, AdvancedControlBase):
+    # This ControlNet is more of an attention patch than a traditional controlnet
+    def __init__(self, patch_attn1: LLLitePatch, patch_attn2: LLLitePatch, timestep_keyframes: TimestepKeyframeGroup, device=None):
+        super().__init__(device)
+        AdvancedControlBase.__init__(self, super(), timestep_keyframes=timestep_keyframes, weights_default=ControlWeights.controllllite(), require_model=True)
+        self.patch_attn1 = patch_attn1.set_control(self)
+        self.patch_attn2 = patch_attn2.set_control(self)
+        self.latent_dims_div2 = None
+        self.latent_dims_div4 = None
+
+    def patch_model(self, model: ModelPatcher):
+        model.set_model_attn1_patch(self.patch_attn1)
+        model.set_model_attn2_patch(self.patch_attn2)
+
+    def set_cond_hint(self, *args, **kwargs):
+        to_return = super().set_cond_hint(*args, **kwargs)
+        # cond hint for LLLite needs to be scaled between (-1, 1) instead of (0, 1)
+        self.cond_hint_original = self.cond_hint_original * 2.0 - 1.0
+        return to_return
+
+    def pre_run_advanced(self, *args, **kwargs):
+        AdvancedControlBase.pre_run_advanced(self, *args, **kwargs)
+        #logger.error(f"in cn: {id(self.patch_attn1)},{id(self.patch_attn2)}")
+        self.patch_attn1.set_control(self)
+        self.patch_attn2.set_control(self)
+        #logger.warn(f"in pre_run_advanced: {id(self)}")
+    
+    def get_control_advanced(self, x_noisy: Tensor, t, cond, batched_number: int):
+        # normal ControlNet stuff
+        control_prev = None
+        if self.previous_controlnet is not None:
+            control_prev = self.previous_controlnet.get_control(x_noisy, t, cond, batched_number)
+
+        if self.timestep_range is not None:
+            if t[0] > self.timestep_range[0] or t[0] < self.timestep_range[1]:
+                return control_prev
+        
+        dtype = x_noisy.dtype
+        # prepare cond_hint
+        if self.sub_idxs is not None or self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
+            if self.cond_hint is not None:
+                del self.cond_hint
+            self.cond_hint = None
+            # if self.cond_hint_original length greater or equal to real latent count, subdivide it before scaling
+            if self.sub_idxs is not None and self.cond_hint_original.size(0) >= self.full_latent_length:
+                self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original[self.sub_idxs], x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(dtype).to(self.device)
+            else:
+                self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(dtype).to(self.device)
+        if x_noisy.shape[0] != self.cond_hint.shape[0]:
+            self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number)
+        # some special logic here compared to other controlnets:
+        # * The cond_emb in attn patches will divide latent dims by 2 or 4, integer
+        # * Due to this loss, the cond_emb will become smaller than x input if latent dims are not divisble by 2 or 4
+        divisible_by_2_h = x_noisy.shape[2]%2==0
+        divisible_by_2_w = x_noisy.shape[3]%2==0
+        if not (divisible_by_2_h and divisible_by_2_w):
+            #logger.warn(f"{x_noisy.shape} not divisible by 2!")
+            new_h = (x_noisy.shape[2]//2)*2
+            new_w = (x_noisy.shape[3]//2)*2
+            if not divisible_by_2_h:
+                new_h += 2
+            if not divisible_by_2_w:
+                new_w += 2
+            self.latent_dims_div2 = (new_h, new_w)
+        divisible_by_4_h = x_noisy.shape[2]%4==0
+        divisible_by_4_w =  x_noisy.shape[3]%4==0
+        if not (divisible_by_4_h and divisible_by_4_w):
+            #logger.warn(f"{x_noisy.shape} not divisible by 4!")
+            new_h = (x_noisy.shape[2]//4)*4
+            new_w = (x_noisy.shape[3]//4)*4
+            if not divisible_by_4_h:
+                new_h += 4
+            if not divisible_by_4_w:
+                new_w += 4
+            self.latent_dims_div4 = (new_h, new_w)
+        # prepare mask
+        self.prepare_mask_cond_hint(x_noisy=x_noisy, t=t, cond=cond, batched_number=batched_number)
+        # done preparing; model patches will take care of everything now.
+        # return normal controlnet stuff
+        return control_prev
    
    def cleanup_advanced(self):
        super().cleanup_advanced()
-        self.patch.cleanup()
+        self.patch_attn1.cleanup()
+        self.patch_attn2.cleanup()
+        self.latent_dims_div2 = None
+        self.latent_dims_div4 = None
    
    def copy(self):
-        c = ControlLLLiteAdvanced(self.patch, self.timestep_keyframes)
+        c = ControlLLLiteAdvanced(self.patch_attn1, self.patch_attn2, self.timestep_keyframes)
        self.copy_to(c)
        self.copy_to_advanced(c)
        return c
    
    # deepcopy needs to properly keep track of objects to work between model.clone calls!
-    def __deepcopy__(self, *args, **kwargs):
-        return self
+    # def __deepcopy__(self, *args, **kwargs):
+    #     self.cleanup_advanced()
+    #     return self

    # def get_models(self):
    #     # get_models is called once at the start of every KSampler run - use to reset already_patched status
    #     out = super().get_models()
+    #     logger.error(f"in get_models! {id(self)}")
    #     return out


@@ -360,8 +515,9 @@ def load_controlnet(ckpt_path, timestep_keyframe: TimestepKeyframeGroup=None, mo
    controlnet_type = ControlWeightType.DEFAULT
    has_controlnet_key = False
    has_motion_modules_key = False
+    has_temporal_res_block_key = False
    for key in controlnet_data:
-        # LLLLite check
+        # LLLite check
        if "lllite" in key:
            controlnet_type = ControlWeightType.CONTROLLLLITE
            break
@@ -370,14 +526,23 @@ def load_controlnet(ckpt_path, timestep_keyframe: TimestepKeyframeGroup=None, mo
            has_motion_modules_key = True
        elif "controlnet" in key:
            has_controlnet_key = True
+        # SVD-ControlNet check
+        elif "temporal_res_block" in key:
+            has_temporal_res_block_key = True
    if has_controlnet_key and has_motion_modules_key:
        controlnet_type = ControlWeightType.SPARSECTRL
+    elif has_controlnet_key and has_temporal_res_block_key:
+        controlnet_type = ControlWeightType.SVD_CONTROLNET

    if controlnet_type != ControlWeightType.DEFAULT:
        if controlnet_type == ControlWeightType.CONTROLLLLITE:
            control = load_controllllite(ckpt_path, controlnet_data=controlnet_data, timestep_keyframe=timestep_keyframe)
        elif controlnet_type == ControlWeightType.SPARSECTRL:
            control = load_sparsectrl(ckpt_path, controlnet_data=controlnet_data, timestep_keyframe=timestep_keyframe, model=model)
+        elif controlnet_type == ControlWeightType.SVD_CONTROLNET:
+            control = load_svdcontrolnet(ckpt_path, controlnet_data=controlnet_data, timestep_keyframe=timestep_keyframe)
+            #raise Exception(f"SVD-ControlNet is not supported yet!")
+            #control = comfy_cn.load_controlnet(ckpt_path, model=model)
    # otherwise, load vanilla ControlNet
    else:
        try:
@@ -602,6 +767,129 @@ def load_controllllite(ckpt_path: str, controlnet_data: dict[str, Tensor]=None,

    #logger.info(f"loaded {ckpt_path} successfully, {len(modules)} modules")

-    patch = LLLitePatch(modules=modules)
-    control = ControlLLLiteAdvanced(patch=patch, timestep_keyframes=timestep_keyframe)
+    patch_attn1 = LLLitePatch(modules=modules, patch_type=LLLitePatch.ATTN1)
+    patch_attn2 = LLLitePatch(modules=modules, patch_type=LLLitePatch.ATTN2)
+    control = ControlLLLiteAdvanced(patch_attn1=patch_attn1, patch_attn2=patch_attn2, timestep_keyframes=timestep_keyframe)
    return control
+
+
+def load_svdcontrolnet(ckpt_path: str, controlnet_data: dict[str, Tensor]=None, timestep_keyframe: TimestepKeyframeGroup=None, model=None):
+    if controlnet_data is None:
+        controlnet_data = comfy.utils.load_torch_file(ckpt_path, safe_load=True)
+
+    controlnet_config = None
+    if "controlnet_cond_embedding.conv_in.weight" in controlnet_data: #diffusers format
+        unet_dtype = comfy.model_management.unet_dtype()
+        controlnet_config = svd_unet_config_from_diffusers_unet(controlnet_data, unet_dtype)
+        diffusers_keys = svd_unet_to_diffusers(controlnet_config)
+        diffusers_keys["controlnet_mid_block.weight"] = "middle_block_out.0.weight"
+        diffusers_keys["controlnet_mid_block.bias"] = "middle_block_out.0.bias"
+
+        count = 0
+        loop = True
+        while loop:
+            suffix = [".weight", ".bias"]
+            for s in suffix:
+                k_in = "controlnet_down_blocks.{}{}".format(count, s)
+                k_out = "zero_convs.{}.0{}".format(count, s)
+                if k_in not in controlnet_data:
+                    loop = False
+                    break
+                diffusers_keys[k_in] = k_out
+            count += 1
+
+        count = 0
+        loop = True
+        while loop:
+            suffix = [".weight", ".bias"]
+            for s in suffix:
+                if count == 0:
+                    k_in = "controlnet_cond_embedding.conv_in{}".format(s)
+                else:
+                    k_in = "controlnet_cond_embedding.blocks.{}{}".format(count - 1, s)
+                k_out = "input_hint_block.{}{}".format(count * 2, s)
+                if k_in not in controlnet_data:
+                    k_in = "controlnet_cond_embedding.conv_out{}".format(s)
+                    loop = False
+                diffusers_keys[k_in] = k_out
+            count += 1
+
+        new_sd = {}
+        for k in diffusers_keys:
+            if k in controlnet_data:
+                new_sd[diffusers_keys[k]] = controlnet_data.pop(k)
+
+        leftover_keys = controlnet_data.keys()
+        if len(leftover_keys) > 0:
+            spatial_leftover_keys = []
+            temporal_leftover_keys = []
+            other_leftover_keys = []
+            for key in leftover_keys:
+                if "spatial" in key:
+                    spatial_leftover_keys.append(key)
+                elif "temporal" in key:
+                    temporal_leftover_keys.append(key)
+                else:
+                    other_leftover_keys.append(key)
+            logger.warn(f"spatial_leftover_keys ({len(spatial_leftover_keys)}): {spatial_leftover_keys}")
+            logger.warn(f"temporal_leftover_keys ({len(temporal_leftover_keys)}): {temporal_leftover_keys}")
+            logger.warn(f"other_leftover_keys ({len(other_leftover_keys)}): {other_leftover_keys}")
+            #print("leftover keys:", leftover_keys)
+        controlnet_data = new_sd
+
+    pth_key = 'control_model.zero_convs.0.0.weight'
+    pth = False
+    key = 'zero_convs.0.0.weight'
+    if pth_key in controlnet_data:
+        pth = True
+        key = pth_key
+        prefix = "control_model."
+    elif key in controlnet_data:
+        prefix = ""
+    else:
+        raise ValueError("The provided model is not a valid SVD-ControlNet model! [ErrorCode: MUSTARD]")
+
+    if controlnet_config is None:
+        unet_dtype = comfy.model_management.unet_dtype()
+        controlnet_config = comfy.model_detection.model_config_from_unet(controlnet_data, prefix, unet_dtype, True).unet_config
+    load_device = comfy.model_management.get_torch_device()
+    manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device)
+    if manual_cast_dtype is not None:
+        controlnet_config["operations"] = comfy.ops.manual_cast
+    controlnet_config.pop("out_channels")
+    controlnet_config["hint_channels"] = controlnet_data["{}input_hint_block.0.weight".format(prefix)].shape[1]
+    control_model = SVDControlNet(**controlnet_config)
+
+    if pth:
+        if 'difference' in controlnet_data:
+            if model is not None:
+                comfy.model_management.load_models_gpu([model])
+                model_sd = model.model_state_dict()
+                for x in controlnet_data:
+                    c_m = "control_model."
+                    if x.startswith(c_m):
+                        sd_key = "diffusion_model.{}".format(x[len(c_m):])
+                        if sd_key in model_sd:
+                            cd = controlnet_data[x]
+                            cd += model_sd[sd_key].type(cd.dtype).to(cd.device)
+            else:
+                print("WARNING: Loaded a diff controlnet without a model. It will very likely not work.")
+
+        class WeightsLoader(torch.nn.Module):
+            pass
+        w = WeightsLoader()
+        w.control_model = control_model
+        missing, unexpected = w.load_state_dict(controlnet_data, strict=False)
+    else:
+        missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False)
+    if len(missing) > 0 or len(unexpected) > 0:
+        logger.info(f"SVD-ControlNet: {missing}, {unexpected}")
+
+    global_average_pooling = False
+    filename = os.path.splitext(ckpt_path)[0]
+    if filename.endswith("_shuffle") or filename.endswith("_shuffle_fp16"): #TODO: smarter way of enabling global_average_pooling
+        global_average_pooling = True
+
+    control = SVDControlNetAdvanced(control_model, timestep_keyframes=timestep_keyframe, global_average_pooling=global_average_pooling, load_device=load_device, manual_cast_dtype=manual_cast_dtype)
+    return control
+
@@ -11,7 +11,7 @@ import comfy.utils
 from comfy.controlnet import ControlBase

 from .logger import logger
-from .utils import AdvancedControlBase, prepare_mask_batch
+from .utils import AdvancedControlBase, deepcopy_with_sharing, prepare_mask_batch


 def extra_options_to_module_prefix(extra_options):
@@ -38,12 +38,16 @@ def extra_options_to_module_prefix(extra_options):


 class LLLitePatch:
-    def __init__(self, modules: dict[str, 'LLLiteModule'], control: Union[AdvancedControlBase, ControlBase]=None):
+    ATTN1 = "attn1"
+    ATTN2 = "attn2"
+    def __init__(self, modules: dict[str, 'LLLiteModule'], patch_type: str, control: Union[AdvancedControlBase, ControlBase]=None):
        self.modules = modules
        self.control = control
+        self.patch_type = patch_type
        #logger.error(f"create LLLitePatch: {id(self)},{control}")
    
    def __call__(self, q, k, v, extra_options):
+        #logger.error(f"in __call__: {id(self)}")
        # determine if have anything to run
        if self.control.timestep_range is not None:
            # it turns out comparing single-value tensors to floats is extremely slow
@@ -78,21 +82,37 @@ class LLLitePatch:
            self.modules[d] = self.modules[d].to(device)
        return self
    
-    def set_control(self, control: Union[AdvancedControlBase, ControlBase]):
+    def set_control(self, control: Union[AdvancedControlBase, ControlBase]) -> 'LLLitePatch':
        self.control = control
-        #logger.error(f"set control for LLLitePatch: {id(self)},{id(control)}")
+        return self
+        #logger.error(f"set control for LLLitePatch: {id(self)}, cn: {id(control)}")

    def clone_with_control(self, control: AdvancedControlBase):
        #logger.error(f"clone-set control for LLLitePatch: {id(self)},{id(control)}")
-        return LLLitePatch(self.modules, control)
+        return LLLitePatch(self.modules, self.patch_type, control)

    def cleanup(self):
-        #del self.control
-        #self.control = None
+        #total_cleaned = 0
        for module in self.modules.values():
            module.cleanup()
+        #    total_cleaned += 1
+        #logger.info(f"cleaned modules: {total_cleaned}, {id(self)}")
        #logger.error(f"cleanup LLLitePatch: {id(self)}")

+    # make sure deepcopy does not copy control, and deepcopied LLLitePatch should be assigned to control
+    def __deepcopy__(self, memo):
+        self.cleanup()
+        to_return: LLLitePatch = deepcopy_with_sharing(self, shared_attribute_names = ['control'], memo=memo)
+        #logger.warn(f"patch {id(self)} turned into {id(to_return)}")
+        try:
+            if self.patch_type == self.ATTN1:
+                to_return.control.patch_attn1 = to_return
+            elif self.patch_type == self.ATTN2:
+                to_return.control.patch_attn2 = to_return
+        except Exception:
+            pass
+        return to_return
+

 # TODO: use comfy.ops to support fp8 properly
 class LLLiteModule(torch.nn.Module):
@@ -159,6 +179,7 @@ class LLLiteModule(torch.nn.Module):
        self.prev_sub_idxs = None

    def cleanup(self):
+        del self.cond_emb
        self.cond_emb = None
        self.cx_shape = None
        self.prev_batch = 0
@@ -167,9 +188,15 @@ class LLLiteModule(torch.nn.Module):
    def forward(self, x: Tensor, control: Union[AdvancedControlBase, ControlBase]):
        mask = None
        mask_tk = None
+        #logger.info(x.shape)
        if self.cond_emb is None or control.sub_idxs != self.prev_sub_idxs or x.shape[0] != self.prev_batch:
            # print(f"cond_emb is None, {self.name}")
-            cx = self.conditioning1(control.cond_hint.to(x.device, dtype=x.dtype))
+            cond_hint = control.cond_hint.to(x.device, dtype=x.dtype)
+            if control.latent_dims_div2 is not None and x.shape[-1] != 1280:
+                cond_hint = comfy.utils.common_upscale(cond_hint, control.latent_dims_div2[0] * 8, control.latent_dims_div2[1] * 8, 'nearest-exact', "center").to(x.device, dtype=x.dtype)
+            elif control.latent_dims_div4 is not None and x.shape[-1] == 1280:
+                cond_hint = comfy.utils.common_upscale(cond_hint, control.latent_dims_div4[0] * 8, control.latent_dims_div4[1] * 8, 'nearest-exact', "center").to(x.device, dtype=x.dtype)
+            cx = self.conditioning1(cond_hint)
            self.cx_shape = cx.shape
            if not self.is_conv2d:
                # reshape / b,c,h,w -> b,h*w,c
@@ -211,6 +238,7 @@ class LLLiteModule(torch.nn.Module):
        elif mask_tk is not None:
            mask = mask * mask_tk

+        #logger.info(f"cs: {cx.shape}, x: {x.shape}, is_conv2d: {self.is_conv2d}")
        cx = torch.cat([cx, self.down(x)], dim=1 if self.is_conv2d else 2)
        cx = self.mid(cx)
        cx = self.up(cx)
@@ -4,7 +4,7 @@ import torch

 import numpy as np
 from PIL import Image, ImageOps
-from .utils import ControlWeights, LatentKeyframeGroup, TimestepKeyframeGroup, TimestepKeyframe
+from .utils import ControlWeights, LatentKeyframeGroup, TimestepKeyframeGroup, TimestepKeyframe, BIGMAX
 from .logger import logger


@@ -16,8 +16,8 @@ class LoadImagesFromDirectory:
                "directory": ("STRING", {"default": ""}),
            },
            "optional": {
-                "image_load_cap": ("INT", {"default": 0, "min": 0, "step": 1}),
-                "start_index": ("INT", {"default": 0, "min": 0, "step": 1}),
+                "image_load_cap": ("INT", {"default": 0, "min": 0, "max": BIGMAX, "step": 1}),
+                "start_index": ("INT", {"default": 0, "min": 0, "max": BIGMAX, "step": 1}),
            }
        }
    
@@ -2,7 +2,7 @@ from typing import Union
 import numpy as np
 from collections.abc import Iterable

-from .utils import LatentKeyframe, LatentKeyframeGroup
+from .utils import LatentKeyframe, LatentKeyframeGroup, BIGMIN, BIGMAX
 from .utils import StrengthInterpolation as SI
 from .logger import logger

@@ -12,7 +12,7 @@ class LatentKeyframeNode:
    def INPUT_TYPES(s):
        return {
            "required": {
-                "batch_index": ("INT", {"default": 0, "min": -1000, "max": 1000, "step": 1}),
+                "batch_index": ("INT", {"default": 0, "min": BIGMIN, "max": BIGMAX, "step": 1}),
                "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001}, ),
            },
            "optional": {
@@ -163,8 +163,8 @@ class LatentKeyframeInterpolationNode:
    def INPUT_TYPES(s):
        return {
            "required": {
-                "batch_index_from": ("INT", {"default": 0, "min": -10000, "max": 10000, "step": 1}),
-                "batch_index_to_excl": ("INT", {"default": 0, "min": -10000, "max": 10000, "step": 1}),
+                "batch_index_from": ("INT", {"default": 0, "min": BIGMIN, "max": BIGMAX, "step": 1}),
+                "batch_index_to_excl": ("INT", {"default": 0, "min": BIGMIN, "max": BIGMAX, "step": 1}),
                "strength_from": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001}, ),
                "strength_to": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001}, ),
                "interpolation": ([SI.LINEAR, SI.EASE_IN, SI.EASE_OUT, SI.EASE_IN_OUT], ),
@@ -3,6 +3,7 @@ from torch import Tensor
 import folder_paths
 from nodes import VAEEncode
 import comfy.utils
+from comfy.sd import VAE

 from .utils import TimestepKeyframeGroup
 from .control_sparsectrl import SparseMethod, SparseIndexMethod, SparseSettings, SparseSpreadMethod, PreprocSparseRGBWrapper
@@ -148,12 +149,15 @@ class RgbSparseCtrlPreprocessor:

    CATEGORY = "Adv-ControlNet 🛂🅐🅒🅝/SparseCtrl/preprocess"

-    def preprocess_images(self, vae, image: Tensor, latent_size: Tensor):
+    def preprocess_images(self, vae: VAE, image: Tensor, latent_size: Tensor):
        # first, resize image to match latents
        image = image.movedim(-1,1)
        image = comfy.utils.common_upscale(image, latent_size["samples"].shape[3] * 8, latent_size["samples"].shape[2] * 8, 'nearest-exact', "center")
        image = image.movedim(1,-1)
        # then, vae encode
-        image = VAEEncode.vae_encode_crop_pixels(image)
+        try:
+            image = vae.vae_encode_crop_pixels(image)
+        except Exception:
+            image = VAEEncode.vae_encode_crop_pixels(image)
        encoded = vae.encode(image[:,:,:,:3])
        return (PreprocSparseRGBWrapper(condhint=encoded),)
@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import Callable, Union
 import torch
 from torch import Tensor
@@ -10,6 +11,9 @@ from comfy.model_patcher import ModelPatcher

 from .logger import logger

+BIGMIN = -(2**63-1)
+BIGMAX = (2**63-1)
+
 def load_torch_file_with_dict_factory(controlnet_data: dict[str, Tensor], orig_load_torch_file: Callable):
    def load_torch_file_with_dict(*args, **kwargs):
        # immediately restore load_torch_file to original version
@@ -34,6 +38,7 @@ class ControlWeightType:
    CONTROLNET = "controlnet"
    CONTROLLORA = "controllora"
    CONTROLLLLITE = "controllllite"
+    SVD_CONTROLNET = "svd_controlnet"
    SPARSECTRL = "sparsectrl"


@@ -259,6 +264,44 @@ def linear_conversion(x, x_min=0.0, x_max=1.0, new_min=0.0, new_max=1.0):
    return (((x - x_min)/(x_max - x_min)) * (new_max - new_min)) + new_min


+# from https://stackoverflow.com/a/24621200
+def deepcopy_with_sharing(obj, shared_attribute_names, memo=None):
+    '''
+    Deepcopy an object, except for a given list of attributes, which should
+    be shared between the original object and its copy.
+
+    obj is some object
+    shared_attribute_names: A list of strings identifying the attributes that
+        should be shared between the original and its copy.
+    memo is the dictionary passed into __deepcopy__.  Ignore this argument if
+        not calling from within __deepcopy__.
+    '''
+    assert isinstance(shared_attribute_names, (list, tuple))
+
+    shared_attributes = {k: getattr(obj, k) for k in shared_attribute_names}
+
+    if hasattr(obj, '__deepcopy__'):
+        # Do hack to prevent infinite recursion in call to deepcopy
+        deepcopy_method = obj.__deepcopy__
+        obj.__deepcopy__ = None
+
+    for attr in shared_attribute_names:
+        del obj.__dict__[attr]
+
+    clone = deepcopy(obj)
+
+    for attr, val in shared_attributes.items():
+        setattr(obj, attr, val)
+        setattr(clone, attr, val)
+
+    if hasattr(obj, '__deepcopy__'):
+        # Undo hack
+        obj.__deepcopy__ = deepcopy_method
+        del clone.__deepcopy__
+
+    return clone
+
+
 class WeightTypeException(TypeError):
    "Raised when weight not compatible with AdvancedControlBase object"
    pass