From 3fe4de9356a3356c095c2b1db9eb2b9082063a45 Mon Sep 17 00:00:00 2001 From: CrossProduct Date: Tue, 4 Feb 2025 23:03:42 +0000 Subject: [PATCH] Modifed the inference loop to try something. also added a copy of the configuration. --- finetrainers/dataset.py | 35 ++++++------- finetrainers/trainer.py | 7 +-- memorize.json | 110 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 20 deletions(-) create mode 100644 memorize.json diff --git a/finetrainers/dataset.py b/finetrainers/dataset.py index c1e8756..638c1d4 100644 --- a/finetrainers/dataset.py +++ b/finetrainers/dataset.py @@ -144,11 +144,11 @@ class ImageOrVideoDataset(Dataset): if video_path.suffix.lower() in [".png", ".jpg", ".jpeg"]: video = self._preprocess_image(video_path) else: - video = self._preprocess_video(video_path) + video,_ = self._preprocess_video(video_path) if self.pose_column != None: - pose = self._preprocess_video(self.pose_paths[index]) - img_ref = self._preprocess_video_image_reference_video(video_path) + pose,num_frames = self._preprocess_video(self.pose_paths[index]) + img_ref = self._preprocess_video_image_reference_video(video_path,num_frames) return { "prompt": prompt, @@ -278,29 +278,30 @@ class ImageOrVideoDataset(Dataset): frames = frames[: self.max_num_frames].float() frames = frames.permute(0, 3, 1, 2).contiguous() frames = torch.stack([self.video_transforms(frame) for frame in frames], dim=0) - return frames + return frames,len(frames) - def _preprocess_video_image_reference_video(self, path: Path) -> torch.Tensor: + # use the frame_length of the pose video. + def _preprocess_video_image_reference_video(self, path: Path,frame_length:int) -> torch.Tensor: video_reader = decord.VideoReader(uri=path.as_posix()) video_num_frames = len(video_reader) - nearest_frame_bucket = min( - [bucket for bucket in self.resolution_buckets if bucket[0] <= video_num_frames], - key=lambda x: abs(x[0] - min(video_num_frames, self.max_num_frames)), - default=1, - )[0] + # nearest_frame_bucket = min( + # [bucket for bucket in self.resolution_buckets if bucket[0] <= video_num_frames], + # key=lambda x: abs(x[0] - min(video_num_frames, self.max_num_frames)), + # default=1, + # )[0] - frame_indices = [0 for _ in range(video_num_frames)] + frame_indices = [0 for _ in range(frame_length)] frames = video_reader.get_batch(frame_indices) - frames = frames[:nearest_frame_bucket].float() + # frames = frames[:nearest_frame_bucket].float() frames = frames.permute(0, 3, 1, 2).contiguous() - nearest_res = self._find_nearest_resolution(frames.shape[2], frames.shape[3]) - frames_resized = torch.stack([resize(frame, nearest_res) for frame in frames], dim=0) - frames = torch.stack([self.video_transforms(frame) for frame in frames_resized], dim=0) + # nearest_res = self._find_nearest_resolution(frames.shape[2], frames.shape[3]) + # frames_resized = torch.stack([resize(frame, nearest_res) for frame in frames], dim=0) + # frames = torch.stack([self.video_transforms(frame) for frame in frames_resized], dim=0) # torch.Size([96, 3, 512, 768]) - + frames = torch.stack([self.video_transforms(frame) for frame in frames], dim=0) return frames @@ -341,7 +342,7 @@ class ImageOrVideoDatasetWithResizing(ImageOrVideoDataset): frames_resized = torch.stack([resize(frame, nearest_res) for frame in frames], dim=0) frames = torch.stack([self.video_transforms(frame) for frame in frames_resized], dim=0) - return frames + return frames,len(frames) def _find_nearest_resolution(self, height, width): nearest_res = min(self.resolution_buckets, key=lambda x: abs(x[1] - height) + abs(x[2] - width)) diff --git a/finetrainers/trainer.py b/finetrainers/trainer.py index e89565b..eaffea8 100644 --- a/finetrainers/trainer.py +++ b/finetrainers/trainer.py @@ -1039,10 +1039,11 @@ class Trainer: # loading videos for inference .. if self.pose_condition: if pose_video is not None: - pose_video = self.dataset._preprocess_video(Path(pose_video)).unsqueeze(0) #self.preprocess_condition_video(pose_video,max_num_frames=self.dataset.max_num_frames) + pose_video,num_frames = self.dataset._preprocess_video(Path(pose_video)) #self.preprocess_condition_video(pose_video,max_num_frames=self.dataset.max_num_frames) + pose_video = pose_video.unsqueeze(0) if img_ref_video is not None: - img_ref_video = self.dataset._preprocess_video_image_reference_video(Path(img_ref_video)).unsqueeze(0) #self.preprocess_condition_video_image_reference_video(path=img_ref_video,resolution_buckets=self.dataset.resolution_buckets,max_num_frames=self.dataset.max_num_frames) - + img_ref_video = self.dataset._preprocess_video_image_reference_video(Path(img_ref_video),num_frames) #self.preprocess_condition_video_image_reference_video(path=img_ref_video,resolution_buckets=self.dataset.resolution_buckets,max_num_frames=self.dataset.max_num_frames) + img_ref_video = img_ref_video.unsqueeze(0) logger.debug( f"Validating sample {i + 1}/{num_validation_samples} on process {accelerator.process_index}. Prompt: {prompt}", main_process_only=False, diff --git a/memorize.json b/memorize.json new file mode 100644 index 0000000..0a40789 --- /dev/null +++ b/memorize.json @@ -0,0 +1,110 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug train.py", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/train.py", + "console": "integratedTerminal", + "env": { + "WANDB_MODE": "online", + "NCCL_P2P_DISABLE": "1", + "TORCH_NCCL_ENABLE_MONITORING": "0", + "FINETRAINERS_LOG_LEVEL": "DEBUG", + "CUDA_VISIBLE_DEVICES": "0" + }, + "args": [ + "--model_name", + "ltx_video", + "--pretrained_model_name_or_path", + "Lightricks/LTX-Video", + "--data_root", + "${workspaceFolder}/video-dataset-pose-test", + "--video_column", + "videos.txt", + "--caption_column", + "prompts.txt", + "--pose_column", + "poses.txt", + "--flow_weighting_scheme", + "logit_normal", + "--id_token", + "", + "--video_resolution_buckets", + "96x512x768", + "--caption_dropout_p", + "0.05", + "--dataloader_num_workers", + "0", + "--flow_resolution_shifting", + "--training_type", + "full-finetune", + "--seed", + "42", + "--mixed_precision", + "bf16", + "--batch_size", + "1", + "--train_steps", + "100000", + "--target_modules", + "to_q", + "to_k", + "to_v", + "to_out.0", + "--gradient_accumulation_steps", + "1", + "--gradient_checkpointing", + "--checkpointing_steps", + "1000", + "--checkpointing_limit", + "2", + "--enable_slicing", + "--enable_tiling", + "--optimizer", + "adamw", + "--lr", + "1e-4", + "--validation_frame_rate", + "24", + "--lr_scheduler", + "constant_with_warmup", + "--lr_warmup_steps", + "100", + "--lr_num_cycles", + "1", + "--beta1", + "0.9", + "--beta2", + "0.95", + "--weight_decay", + "1e-4", + "--epsilon", + "1e-8", + "--max_grad_norm", + "1.0", + "--validation_prompts", + "A beautiful woman wearing a very revealing red lingerie thong and bra, walks through a modern, minimalist interior featuring a white sofa, black chairs, and a glass partition. Her confident and alluring demeanor is highlighted by the soft lighting and the room's neutral tones. As she strides forward, her bare feet touch the carpet, and her brown hair cascades over her shoulders. The setting includes a sleek white dining table, and black dining table chairs, a couch to the left, and a bookshelf filled with books and decorative items.@@@96x512x768", + "--validation_pose_videos", + "/home/ubuntu/upstream/finetrainers/video-dataset-pose-test/poses/stasyq-375-Scene-021-pose.mp4", + "--validation_img_ref_videos", + "/home/ubuntu/upstream/finetrainers/video-dataset-pose-test/videos/stasyq-375-Scene-021.mp4", + "--num_validation_videos", + "1", + "--validation_steps", + "500", + "--tracker_name", + "finetrainers-ltxv", + "--output_dir", + "ltx-video/ltxv_strip", + "--nccl_timeout", + "1800", + "--report_to", + "wandb", + "--resume_from_checkpoint=latest" + ], + "justMyCode": false + } + ] +} \ No newline at end of file