From a16285141e68cf746535d48c2bf5e0f3917da2bc Mon Sep 17 00:00:00 2001 From: CrossProduct Date: Wed, 29 Jan 2025 05:30:17 +0000 Subject: [PATCH] Noted place that requires intervention. --- finetrainers/conditioning/conditioned_pipeline.py | 6 +++--- finetrainers/ltx_video/full_finetune_condition.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/finetrainers/conditioning/conditioned_pipeline.py b/finetrainers/conditioning/conditioned_pipeline.py index b52e845..2067b51 100644 --- a/finetrainers/conditioning/conditioned_pipeline.py +++ b/finetrainers/conditioning/conditioned_pipeline.py @@ -279,16 +279,16 @@ class LTXConditionedPipeline(LTXPipeline): continue # change this # latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents - # latent_model_input = latent_model_input.to(prompt_embeds.dtype) + #latent_model_input = latent_model_input.to(prompt_embeds.dtype) # might be the bug .. - latent_model_input = noisy_latent_tokens + latent_model_input = torch.cat([noisy_latent_tokens] * 2) if self.do_classifier_free_guidance else noisy_latent_tokens latent_model_input = latent_model_input.to(prompt_embeds.dtype) # broadcast to batch dimension in a way that's compatible with ONNX/Core ML timestep = t.expand(latent_model_input.shape[0]) - +#encoder hidden states are different... noise_pred = self.transformer( hidden_states=condition_tokens, encoder_hidden_states=prompt_embeds, diff --git a/finetrainers/ltx_video/full_finetune_condition.py b/finetrainers/ltx_video/full_finetune_condition.py index 3105fe1..1dc70c0 100644 --- a/finetrainers/ltx_video/full_finetune_condition.py +++ b/finetrainers/ltx_video/full_finetune_condition.py @@ -116,6 +116,9 @@ def conditioned_forward_pass( ) -> torch.Tensor: rope_interpolation_scale = [1 / 25, 32, 32] + # encoder_hidden_states=prompt_embeds, + # timestep=timesteps, + # encoder_attention_mask=prompt_attention_mask, denoised_latents = transformer( hidden_states=noisy_latents, encoder_hidden_states=prompt_embeds,