storytold/FineTrainers-Conditioning
2
0
Fork 0
mirror of https://github.com/storytold/FineTrainers-Conditioning.git synced 2026-10-18 07:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Moved the norm back to prepare latents step.

Browse Source 
Now its time to add the adapter layer.
This commit is contained in:
CrossProduct
2025-01-18 04:18:25 +00:00
parent 44df9e3bce
commit 2720d24f62
3 changed files with 4 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
finetrainers/conditioning/condition_latents_prepare.py
+1 -4
View File
@@ -40,8 +40,6 @@ def _pack_latents(latents: torch.Tensor, patch_size: int = 1, patch_size_t: int
def post_conditioned_latent_patchify( def post_conditioned_latent_patchify(
latents: torch.Tensor, latents: torch.Tensor,
latents_mean: torch.Tensor,
latents_std: torch.Tensor,
num_frames: int, num_frames: int,
height: int, height: int,
width: int, width: int,
@@ -49,7 +47,6 @@ def post_conditioned_latent_patchify(
patch_size_t: int = 1, patch_size_t: int = 1,
**kwargs, **kwargs,
) -> torch.Tensor: ) -> torch.Tensor:
latents = _normalize_latents(latents, latents_mean, latents_std)
latents = _pack_latents(latents, patch_size, patch_size_t) latents = _pack_latents(latents, patch_size, patch_size_t)
return {"latents": latents, "num_frames": num_frames, "height": height, "width": width} return {"latents": latents, "num_frames": num_frames, "height": height, "width": width}
@@ -76,7 +73,7 @@ def prepare_latents_for_conditioning(
latents = latents.to(dtype=dtype) latents = latents.to(dtype=dtype)
_, _, num_frames, height, width = latents.shape _, _, num_frames, height, width = latents.shape
# latents = _normalize_latents(latents, vae.latents_mean, vae.latents_std) latents = _normalize_latents(latents, vae.latents_mean, vae.latents_std)
# latents = _pack_latents(latents, patch_size, patch_size_t) # latents = _pack_latents(latents, patch_size, patch_size_t)
return {"latents": latents, return {"latents": latents,
"num_frames": num_frames, "num_frames": num_frames,
finetrainers/ltx_video/full_finetune.py
+2 -1
View File
@@ -11,6 +11,7 @@ from .lora import (
prepare_conditions, prepare_conditions,
prepare_latents, prepare_latents,
validation, validation,
collate_fn_t2v_cond
) )
@@ -24,7 +25,7 @@ LTX_VIDEO_T2V_FULL_FINETUNE_CONFIG = {
"prepare_conditions": prepare_conditions, "prepare_conditions": prepare_conditions,
"prepare_latents": prepare_latents, "prepare_latents": prepare_latents,
"post_latent_preparation": post_latent_preparation, "post_latent_preparation": post_latent_preparation,
"collate_fn": collate_fn_t2v, "collate_fn": collate_fn_t2v_cond,
"forward_pass": forward_pass, "forward_pass": forward_pass,
"validation": validation, "validation": validation,
} }
finetrainers/trainer.py
+1 -11
View File
@@ -799,14 +799,6 @@ class Trainer:
if self.pose_condition: if self.pose_condition:
# condition with the pose information # condition with the pose information
# grab noise pre patchification shape
noise = torch.randn(
latent_conditions["latents"].shape,
generator=self.state.generator,
device=accelerator.device,
dtype=weight_dtype,
)
# normalize and patchify # normalize and patchify
# create noise for latent to feed diffusion transformer to get pred noise # create noise for latent to feed diffusion transformer to get pred noise
noisy_latents = (1.0 - sigmas) * latent_conditions["latents"] + sigmas * noise noisy_latents = (1.0 - sigmas) * latent_conditions["latents"] + sigmas * noise
@@ -819,15 +811,13 @@ class Trainer:
# project this turn into patches # project this turn into patches
latent_final = post_conditioned_latent_patchify(latents=latent_final, latent_final = post_conditioned_latent_patchify(latents=latent_final,
latents_mean=(latent_conditions["mean"] + pose_video_latents["mean"] + img_refs_latents["mean"]) / 3.0,
latents_std=(latent_conditions["std"] + pose_video_latents["std"] + img_refs_latents["std"]) / 3.0,
num_frames=latent_conditions["num_frames"], num_frames=latent_conditions["num_frames"],
height=latent_conditions["height"], height=latent_conditions["height"],
width=latent_conditions["width"], width=latent_conditions["width"],
patch_size = 1, patch_size = 1,
patch_size_t = 1) patch_size_t = 1)
latent_conditions.update({"noisy_latents": latent_final}) latent_conditions.update({"noisy_latents": latent_final["latents"]})
else: else:
# Default to flow-matching noise addition # Default to flow-matching noise addition
noisy_latents = (1.0 - sigmas) * latent_conditions["latents"] + sigmas * noise noisy_latents = (1.0 - sigmas) * latent_conditions["latents"] + sigmas * noise