diff --git a/.gitignore b/.gitignore index 0ab43c6..d2b626e 100644 --- a/.gitignore +++ b/.gitignore @@ -163,7 +163,7 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ - +ltx-video/samples # manually added wandb/ *.txt diff --git a/finetrainers/args.py b/finetrainers/args.py index 855173b..767651a 100644 --- a/finetrainers/args.py +++ b/finetrainers/args.py @@ -556,7 +556,7 @@ def _add_dataset_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--pose_column", type=str, - default="text", + default=None, help="The column of the dataset containing the instance prompt for each video. Or, the name of the file in `--data_root` folder containing the line-separated instance prompts.", ) parser.add_argument( diff --git a/finetrainers/dataset.py b/finetrainers/dataset.py index 721730d..f9d7e78 100644 --- a/finetrainers/dataset.py +++ b/finetrainers/dataset.py @@ -46,11 +46,11 @@ class ImageOrVideoDataset(Dataset): data_root: str, caption_column: str, video_column: str, - pose_column: str, resolution_buckets: List[Tuple[int, int, int]], dataset_file: Optional[str] = None, id_token: Optional[str] = None, remove_llm_prefixes: bool = False, + pose_column: str = None, ) -> None: super().__init__() @@ -178,8 +178,20 @@ class ImageOrVideoDataset(Dataset): prompt_path = self.data_root.joinpath(self.caption_column) video_path = self.data_root.joinpath(self.video_column) - pose_path = self.data_root.joinpath(self.pose_column) - + + if self.pose_column != None: + pose_path = self.data_root.joinpath(self.pose_column) + pose_conditioning = True + + if not pose_path.exists() or not pose_path.is_file() or not self.pose_column: + pose_conditioning = False + + if pose_conditioning: + with open(pose_path, "r", encoding="utf-8") as file: + pose_paths = [self.data_root.joinpath(line.strip()) for line in file.readlines() if len(line.strip()) > 0] + else: + pose_paths = None + if not prompt_path.exists() or not prompt_path.is_file(): raise ValueError( "Expected `--caption_column` to be path to a file in `--data_root` containing line-separated text prompts." @@ -188,16 +200,13 @@ class ImageOrVideoDataset(Dataset): raise ValueError( "Expected `--video_column` to be path to a file in `--data_root` containing line-separated paths to video data in the same directory." ) - if not pose_path.exists() or not pose_path.is_file(): - raise ValueError( - "Expected `--pose_column` to be path to a file in `--data_root` containing line-separated paths to video data in the same directory." - ) + with open(prompt_path, "r", encoding="utf-8") as file: prompts = [line.strip() for line in file.readlines() if len(line.strip()) > 0] with open(video_path, "r", encoding="utf-8") as file: video_paths = [self.data_root.joinpath(line.strip()) for line in file.readlines() if len(line.strip()) > 0] - with open(pose_path, "r", encoding="utf-8") as file: - pose_paths = [self.data_root.joinpath(line.strip()) for line in file.readlines() if len(line.strip()) > 0] + + if any(not path.is_file() for path in video_paths): raise ValueError( diff --git a/finetrainers/ltx_video/lora.py b/finetrainers/ltx_video/lora.py index e7c04e6..893d697 100644 --- a/finetrainers/ltx_video/lora.py +++ b/finetrainers/ltx_video/lora.py @@ -181,7 +181,7 @@ def post_latent_preparation( return {"latents": latents, "num_frames": num_frames, "height": height, "width": width} -def collate_fn_t2v(batch: List[List[Dict[str, torch.Tensor]]]) -> Dict[str, torch.Tensor]: +def collate_fn_t2v_cond(batch: List[List[Dict[str, torch.Tensor]]]) -> Dict[str, torch.Tensor]: return { "prompts": [x["prompt"] for x in batch[0]], "videos": torch.stack([x["video"] for x in batch[0]]), @@ -189,6 +189,13 @@ def collate_fn_t2v(batch: List[List[Dict[str, torch.Tensor]]]) -> Dict[str, torc "img_refs": torch.stack([x["img_ref"] for x in batch[0]]) } +def collate_fn_t2v(batch: List[List[Dict[str, torch.Tensor]]]) -> Dict[str, torch.Tensor]: + return { + "prompts": [x["prompt"] for x in batch[0]], + "videos": torch.stack([x["video"] for x in batch[0]]), + } + + def forward_pass( transformer: LTXVideoTransformer3DModel,