From 305a4a994d04f70c8f02894269f2cb28a2ad4e92 Mon Sep 17 00:00:00 2001 From: CrossProduct Date: Fri, 24 Jan 2025 05:21:53 +0000 Subject: [PATCH] Need to pass in image ref video. --- finetrainers/args.py | 14 +++++++++++++- finetrainers/conditioning/conditioned_pipeline.py | 9 +++++++-- finetrainers/ltx_video/lora.py | 7 +++++-- finetrainers/trainer.py | 5 +++++ 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/finetrainers/args.py b/finetrainers/args.py index 767651a..fb79fe5 100644 --- a/finetrainers/args.py +++ b/finetrainers/args.py @@ -316,6 +316,7 @@ class Args: validation_prompts: List[str] = None validation_images: List[str] = None validation_videos: List[str] = None + validation_pose_videos: List[str] = None validation_heights: List[int] = None validation_widths: List[int] = None validation_num_frames: List[int] = None @@ -879,6 +880,14 @@ def _add_validation_arguments(parser: argparse.ArgumentParser) -> None: default=None, help="One or more image path(s)/URLs that is used during validation to verify that the model is learning. Multiple validation paths should be separated by the '--validation_prompt_seperator' string. These should correspond to the order of the validation prompts.", ) + + parser.add_argument( + "--validation_pose_videos", + type=str, + default=None, + help="One or more pose_videos path(s)/URLs that is used during validation to verify that the model is learning. Multiple validation paths should be separated by the '--validation_prompt_seperator' string. These should correspond to the order of the validation prompts.", + ) + parser.add_argument( "--validation_videos", type=str, @@ -1079,6 +1088,9 @@ def _map_to_args_type(args: Dict[str, Any]) -> Args: validation_prompts = args.validation_prompts.split(args.validation_separator) if args.validation_prompts else [] validation_images = args.validation_images.split(args.validation_separator) if args.validation_images else None validation_videos = args.validation_videos.split(args.validation_separator) if args.validation_videos else None + + validation_pose_videos = args.validation_pose_videos.split(args.validation_separator) if args.validation_pose_videos else None + stripped_validation_prompts = [] validation_heights = [] validation_widths = [] @@ -1106,7 +1118,7 @@ def _map_to_args_type(args: Dict[str, Any]) -> Args: result_args.validation_num_frames = validation_num_frames result_args.validation_images = validation_images result_args.validation_videos = validation_videos - + result_args.validation_pose_videos = validation_pose_videos result_args.num_validation_videos_per_prompt = args.num_validation_videos result_args.validation_every_n_epochs = args.validation_epochs result_args.validation_every_n_steps = args.validation_steps diff --git a/finetrainers/conditioning/conditioned_pipeline.py b/finetrainers/conditioning/conditioned_pipeline.py index 167877f..5fae388 100644 --- a/finetrainers/conditioning/conditioned_pipeline.py +++ b/finetrainers/conditioning/conditioned_pipeline.py @@ -111,6 +111,8 @@ class LTXConditionedPipeline(LTXPipeline): prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0) # 4. Prepare latent variables + # it needs to be the size of the image + num_channels_latents = self.transformer.config.in_channels latents = self.prepare_latents( batch_size * num_videos_per_prompt, @@ -122,9 +124,12 @@ class LTXConditionedPipeline(LTXPipeline): device, generator, latents, + ) # creates noise tensor the size suggested by the user + + pose_latents = self.prepare_latents( + ) - - + # 5. Prepare timesteps latent_num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 diff --git a/finetrainers/ltx_video/lora.py b/finetrainers/ltx_video/lora.py index dec3173..d9ae717 100644 --- a/finetrainers/ltx_video/lora.py +++ b/finetrainers/ltx_video/lora.py @@ -276,6 +276,7 @@ def conditional_validation( image: Optional[Image.Image] = None, video: Optional[List[Image.Image]] = None, pose_video: Optional[List[Image.Image]] = None, + image_ref_video: Optional[List[Image.Image]] = None, height: Optional[int] = None, width: Optional[int] = None, num_frames: Optional[int] = None, @@ -294,9 +295,11 @@ def conditional_validation( "generator": generator, "return_dict": True, "output_type": "pil", - "residual_x": None, + "pose_video": pose_video, + "image_ref_video": image_ref_video, } - + # pose template noisey input [cat] img_ref + pose video + # pose template ref video latent + patchify video latent. # img ref patchify video latent + # add them together as input diff --git a/finetrainers/trainer.py b/finetrainers/trainer.py index 45863f4..bb2d2a0 100644 --- a/finetrainers/trainer.py +++ b/finetrainers/trainer.py @@ -1010,6 +1010,7 @@ class Trainer: prompt = self.args.validation_prompts[i] image = self.args.validation_images[i] video = self.args.validation_videos[i] + pose_video = self.args.validation_pose_videos[i] height = self.args.validation_heights[i] width = self.args.validation_widths[i] num_frames = self.args.validation_num_frames[i] @@ -1018,16 +1019,20 @@ class Trainer: image = load_image(image) if video is not None: video = load_video(video) + if pose_video is not None: + pose_video = load_video(pose_video) logger.debug( f"Validating sample {i + 1}/{num_validation_samples} on process {accelerator.process_index}. Prompt: {prompt}", main_process_only=False, ) + validation_artifacts = self.model_config["validation"]( pipeline=pipeline, prompt=prompt, image=image, video=video, + pose_video=pose_video, height=height, width=width, num_frames=num_frames,