From 731f326e86277d36e1d363c079f062828193d9b4 Mon Sep 17 00:00:00 2001 From: CrossProduct Date: Wed, 22 Jan 2025 20:10:01 +0000 Subject: [PATCH] Added residual --- finetrainers/ltx_video/lora.py | 4 +++- finetrainers/trainer.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/finetrainers/ltx_video/lora.py b/finetrainers/ltx_video/lora.py index 424dc47..8f33da0 100644 --- a/finetrainers/ltx_video/lora.py +++ b/finetrainers/ltx_video/lora.py @@ -220,10 +220,11 @@ def conditioned_forward_pass( num_frames: int, height: int, width: int, + noisy_latents_residual:torch.Tensor, **kwargs, ) -> torch.Tensor: rope_interpolation_scale = [1 / 25, 32, 32] - + denoised_latents = transformer( hidden_states=noisy_latents, encoder_hidden_states=prompt_embeds, @@ -234,6 +235,7 @@ def conditioned_forward_pass( width=width, rope_interpolation_scale=rope_interpolation_scale, return_dict=False, + residual_x=noisy_latents_residual, )[0] return {"latents": denoised_latents} diff --git a/finetrainers/trainer.py b/finetrainers/trainer.py index 9c17640..0e87537 100644 --- a/finetrainers/trainer.py +++ b/finetrainers/trainer.py @@ -827,6 +827,7 @@ class Trainer: patch_size_t = 1) # pose information latent latent_conditions.update({"noisy_latents": latent_final["latents"]}) + latent_conditions.update({"noisy_latents_residual":noisy_latents_residual["latents"]}) else: # Default to flow-matching noise addition noisy_latents = (1.0 - sigmas) * latent_conditions["latents"] + sigmas * noise @@ -848,7 +849,6 @@ class Trainer: transformer=self.transformer, scheduler=self.scheduler, timesteps=timesteps, - residual_x=noisy_latents_residual, **latent_conditions, **text_conditions, )