The encoder hidden states and encode attention mask sizes do not match up in the validation pipeline.

finetrainers/conditioning/LTXVideoConditionedTransformer3DModel.py

@@ -106,7 +106,7 @@ class LTXVideoConditionedTransformer3DModel(LTXVideoTransformer3DModel):
             # inject the condition and the residual then project it into the pretrained proj_in
             hidden_states = self.adapter(residual_x=residual_x,
                                         conditioned_x=hidden_states)
-
+            # whats this value ? 
             hidden_states = self.proj_in(hidden_states)
 
             temb, embedded_timestep = self.time_embed(
@@ -134,7 +134,7 @@ class LTXVideoConditionedTransformer3DModel(LTXVideoTransformer3DModel):
                         return custom_forward
 
                     ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}
-                    hidden_states = torch.utils.checkpoint.checkpoint(
+                    hidden_states = torch.utils.checkpoint.checkpoint( # crashes here
                         create_custom_forward(block),
                         hidden_states,
                         encoder_hidden_states,

finetrainers/conditioning/conditioned_pipeline.py

@@ -180,7 +180,7 @@ class LTXConditionedPipeline(LTXPipeline):
             height,
             width,
             num_frames,
-            torch.float32,
+            torch.bfloat16,
             device,
             generator,
         )
@@ -192,7 +192,7 @@ class LTXConditionedPipeline(LTXPipeline):
             patch_size=self.transformer.config.patch_size,
             patch_size_t=self.transformer.config.patch_size_t,
             device=device,
-            dtype=torch.float32,
+            dtype=torch.bfloat16,
             generator=generator,
         )["latents"]
 
@@ -202,7 +202,7 @@ class LTXConditionedPipeline(LTXPipeline):
             patch_size=self.transformer.config.patch_size,
             patch_size_t=self.transformer.config.patch_size_t,
             device=device,
-            dtype=torch.float32,
+            dtype=torch.bfloat16,
             generator=generator,
         )["latents"]
 
@@ -280,6 +280,8 @@ class LTXConditionedPipeline(LTXPipeline):
                 # change this 
                 # latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
                 # latent_model_input = latent_model_input.to(prompt_embeds.dtype)
+                
+                # might be  the bug ..
                 latent_model_input = noisy_latent_tokens
                 latent_model_input = latent_model_input.to(prompt_embeds.dtype)