Need to pass in image ref video.

finetrainers/args.py

@@ -316,6 +316,7 @@ class Args:
     validation_prompts: List[str] = None
     validation_images: List[str] = None
     validation_videos: List[str] = None
+    validation_pose_videos: List[str] = None
     validation_heights: List[int] = None
     validation_widths: List[int] = None
     validation_num_frames: List[int] = None
@@ -879,6 +880,14 @@ def _add_validation_arguments(parser: argparse.ArgumentParser) -> None:
         default=None,
         help="One or more image path(s)/URLs that is used during validation to verify that the model is learning. Multiple validation paths should be separated by the '--validation_prompt_seperator' string. These should correspond to the order of the validation prompts.",
     )
+    
+    parser.add_argument(
+        "--validation_pose_videos",
+        type=str,
+        default=None,
+        help="One or more pose_videos path(s)/URLs that is used during validation to verify that the model is learning. Multiple validation paths should be separated by the '--validation_prompt_seperator' string. These should correspond to the order of the validation prompts.",
+    )
+
     parser.add_argument(
         "--validation_videos",
         type=str,
@@ -1079,6 +1088,9 @@ def _map_to_args_type(args: Dict[str, Any]) -> Args:
     validation_prompts = args.validation_prompts.split(args.validation_separator) if args.validation_prompts else []
     validation_images = args.validation_images.split(args.validation_separator) if args.validation_images else None
     validation_videos = args.validation_videos.split(args.validation_separator) if args.validation_videos else None
+    
+    validation_pose_videos = args.validation_pose_videos.split(args.validation_separator) if args.validation_pose_videos else None
+
     stripped_validation_prompts = []
     validation_heights = []
     validation_widths = []
@@ -1106,7 +1118,7 @@ def _map_to_args_type(args: Dict[str, Any]) -> Args:
     result_args.validation_num_frames = validation_num_frames
     result_args.validation_images = validation_images
     result_args.validation_videos = validation_videos
-
+    result_args.validation_pose_videos = validation_pose_videos
     result_args.num_validation_videos_per_prompt = args.num_validation_videos
     result_args.validation_every_n_epochs = args.validation_epochs
     result_args.validation_every_n_steps = args.validation_steps

finetrainers/conditioning/conditioned_pipeline.py

@@ -111,6 +111,8 @@ class LTXConditionedPipeline(LTXPipeline):
             prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0)
 
         # 4. Prepare latent variables
+        # it needs to be the size of the image 
+        
         num_channels_latents = self.transformer.config.in_channels
         latents = self.prepare_latents(
             batch_size * num_videos_per_prompt,
@@ -122,9 +124,12 @@ class LTXConditionedPipeline(LTXPipeline):
             device,
             generator,
             latents,
+        ) # creates noise tensor the size suggested by the user
+
+        pose_latents = self.prepare_latents(
+
         )
-        
-        
+
 
         # 5. Prepare timesteps
         latent_num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1

finetrainers/ltx_video/lora.py

@@ -276,6 +276,7 @@ def conditional_validation(
     image: Optional[Image.Image] = None,
     video: Optional[List[Image.Image]] = None,
     pose_video: Optional[List[Image.Image]] = None,
+    image_ref_video: Optional[List[Image.Image]] = None,
     height: Optional[int] = None,
     width: Optional[int] = None,
     num_frames: Optional[int] = None,
@@ -294,9 +295,11 @@ def conditional_validation(
         "generator": generator,
         "return_dict": True,
         "output_type": "pil",
-        "residual_x": None,
+        "pose_video": pose_video,
+        "image_ref_video": image_ref_video,
     }
-    
+    # pose template noisey input [cat] img_ref + pose video 
+      
     # pose template ref video latent + patchify video latent.
     # img ref patchify video latent +
     # add them together as input 

finetrainers/trainer.py

@@ -1010,6 +1010,7 @@ class Trainer:
             prompt = self.args.validation_prompts[i]
             image = self.args.validation_images[i]
             video = self.args.validation_videos[i]
+            pose_video = self.args.validation_pose_videos[i]
             height = self.args.validation_heights[i]
             width = self.args.validation_widths[i]
             num_frames = self.args.validation_num_frames[i]
@@ -1018,16 +1019,20 @@ class Trainer:
                 image = load_image(image)
             if video is not None:
                 video = load_video(video)
+            if pose_video is not None:
+                pose_video = load_video(pose_video)
 
             logger.debug(
                 f"Validating sample {i + 1}/{num_validation_samples} on process {accelerator.process_index}. Prompt: {prompt}",
                 main_process_only=False,
             )
+            
             validation_artifacts = self.model_config["validation"](
                 pipeline=pipeline,
                 prompt=prompt,
                 image=image,
                 video=video,
+                pose_video=pose_video,
                 height=height,
                 width=width,
                 num_frames=num_frames,