mirror of
https://github.com/storytold/FineTrainers-Conditioning.git
synced 2026-10-09 00:09:45 +00:00
Modifed the inference loop to try something. also added a copy of the configuration.
This commit is contained in:
CrossProduct
+18
-17
@@ -144,11 +144,11 @@ class ImageOrVideoDataset(Dataset):
|
||||
if video_path.suffix.lower() in [".png", ".jpg", ".jpeg"]:
|
||||
video = self._preprocess_image(video_path)
|
||||
else:
|
||||
video = self._preprocess_video(video_path)
|
||||
video,_ = self._preprocess_video(video_path)
|
||||
|
||||
if self.pose_column != None:
|
||||
pose = self._preprocess_video(self.pose_paths[index])
|
||||
img_ref = self._preprocess_video_image_reference_video(video_path)
|
||||
pose,num_frames = self._preprocess_video(self.pose_paths[index])
|
||||
img_ref = self._preprocess_video_image_reference_video(video_path,num_frames)
|
||||
|
||||
return {
|
||||
"prompt": prompt,
|
||||
@@ -278,29 +278,30 @@ class ImageOrVideoDataset(Dataset):
|
||||
frames = frames[: self.max_num_frames].float()
|
||||
frames = frames.permute(0, 3, 1, 2).contiguous()
|
||||
frames = torch.stack([self.video_transforms(frame) for frame in frames], dim=0)
|
||||
return frames
|
||||
return frames,len(frames)
|
||||
|
||||
def _preprocess_video_image_reference_video(self, path: Path) -> torch.Tensor:
|
||||
# use the frame_length of the pose video.
|
||||
def _preprocess_video_image_reference_video(self, path: Path,frame_length:int) -> torch.Tensor:
|
||||
video_reader = decord.VideoReader(uri=path.as_posix())
|
||||
|
||||
video_num_frames = len(video_reader)
|
||||
nearest_frame_bucket = min(
|
||||
[bucket for bucket in self.resolution_buckets if bucket[0] <= video_num_frames],
|
||||
key=lambda x: abs(x[0] - min(video_num_frames, self.max_num_frames)),
|
||||
default=1,
|
||||
)[0]
|
||||
# nearest_frame_bucket = min(
|
||||
# [bucket for bucket in self.resolution_buckets if bucket[0] <= video_num_frames],
|
||||
# key=lambda x: abs(x[0] - min(video_num_frames, self.max_num_frames)),
|
||||
# default=1,
|
||||
# )[0]
|
||||
|
||||
frame_indices = [0 for _ in range(video_num_frames)]
|
||||
frame_indices = [0 for _ in range(frame_length)]
|
||||
frames = video_reader.get_batch(frame_indices)
|
||||
frames = frames[:nearest_frame_bucket].float()
|
||||
# frames = frames[:nearest_frame_bucket].float()
|
||||
|
||||
frames = frames.permute(0, 3, 1, 2).contiguous()
|
||||
|
||||
nearest_res = self._find_nearest_resolution(frames.shape[2], frames.shape[3])
|
||||
frames_resized = torch.stack([resize(frame, nearest_res) for frame in frames], dim=0)
|
||||
frames = torch.stack([self.video_transforms(frame) for frame in frames_resized], dim=0)
|
||||
# nearest_res = self._find_nearest_resolution(frames.shape[2], frames.shape[3])
|
||||
# frames_resized = torch.stack([resize(frame, nearest_res) for frame in frames], dim=0)
|
||||
# frames = torch.stack([self.video_transforms(frame) for frame in frames_resized], dim=0)
|
||||
# torch.Size([96, 3, 512, 768])
|
||||
|
||||
frames = torch.stack([self.video_transforms(frame) for frame in frames], dim=0)
|
||||
|
||||
return frames
|
||||
|
||||
@@ -341,7 +342,7 @@ class ImageOrVideoDatasetWithResizing(ImageOrVideoDataset):
|
||||
frames_resized = torch.stack([resize(frame, nearest_res) for frame in frames], dim=0)
|
||||
frames = torch.stack([self.video_transforms(frame) for frame in frames_resized], dim=0)
|
||||
|
||||
return frames
|
||||
return frames,len(frames)
|
||||
|
||||
def _find_nearest_resolution(self, height, width):
|
||||
nearest_res = min(self.resolution_buckets, key=lambda x: abs(x[1] - height) + abs(x[2] - width))
|
||||
|
||||
@@ -1039,10 +1039,11 @@ class Trainer:
|
||||
# loading videos for inference ..
|
||||
if self.pose_condition:
|
||||
if pose_video is not None:
|
||||
pose_video = self.dataset._preprocess_video(Path(pose_video)).unsqueeze(0) #self.preprocess_condition_video(pose_video,max_num_frames=self.dataset.max_num_frames)
|
||||
pose_video,num_frames = self.dataset._preprocess_video(Path(pose_video)) #self.preprocess_condition_video(pose_video,max_num_frames=self.dataset.max_num_frames)
|
||||
pose_video = pose_video.unsqueeze(0)
|
||||
if img_ref_video is not None:
|
||||
img_ref_video = self.dataset._preprocess_video_image_reference_video(Path(img_ref_video)).unsqueeze(0) #self.preprocess_condition_video_image_reference_video(path=img_ref_video,resolution_buckets=self.dataset.resolution_buckets,max_num_frames=self.dataset.max_num_frames)
|
||||
|
||||
img_ref_video = self.dataset._preprocess_video_image_reference_video(Path(img_ref_video),num_frames) #self.preprocess_condition_video_image_reference_video(path=img_ref_video,resolution_buckets=self.dataset.resolution_buckets,max_num_frames=self.dataset.max_num_frames)
|
||||
img_ref_video = img_ref_video.unsqueeze(0)
|
||||
logger.debug(
|
||||
f"Validating sample {i + 1}/{num_validation_samples} on process {accelerator.process_index}. Prompt: {prompt}",
|
||||
main_process_only=False,
|
||||
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Debug train.py",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"env": {
|
||||
"WANDB_MODE": "online",
|
||||
"NCCL_P2P_DISABLE": "1",
|
||||
"TORCH_NCCL_ENABLE_MONITORING": "0",
|
||||
"FINETRAINERS_LOG_LEVEL": "DEBUG",
|
||||
"CUDA_VISIBLE_DEVICES": "0"
|
||||
},
|
||||
"args": [
|
||||
"--model_name",
|
||||
"ltx_video",
|
||||
"--pretrained_model_name_or_path",
|
||||
"Lightricks/LTX-Video",
|
||||
"--data_root",
|
||||
"${workspaceFolder}/video-dataset-pose-test",
|
||||
"--video_column",
|
||||
"videos.txt",
|
||||
"--caption_column",
|
||||
"prompts.txt",
|
||||
"--pose_column",
|
||||
"poses.txt",
|
||||
"--flow_weighting_scheme",
|
||||
"logit_normal",
|
||||
"--id_token",
|
||||
"",
|
||||
"--video_resolution_buckets",
|
||||
"96x512x768",
|
||||
"--caption_dropout_p",
|
||||
"0.05",
|
||||
"--dataloader_num_workers",
|
||||
"0",
|
||||
"--flow_resolution_shifting",
|
||||
"--training_type",
|
||||
"full-finetune",
|
||||
"--seed",
|
||||
"42",
|
||||
"--mixed_precision",
|
||||
"bf16",
|
||||
"--batch_size",
|
||||
"1",
|
||||
"--train_steps",
|
||||
"100000",
|
||||
"--target_modules",
|
||||
"to_q",
|
||||
"to_k",
|
||||
"to_v",
|
||||
"to_out.0",
|
||||
"--gradient_accumulation_steps",
|
||||
"1",
|
||||
"--gradient_checkpointing",
|
||||
"--checkpointing_steps",
|
||||
"1000",
|
||||
"--checkpointing_limit",
|
||||
"2",
|
||||
"--enable_slicing",
|
||||
"--enable_tiling",
|
||||
"--optimizer",
|
||||
"adamw",
|
||||
"--lr",
|
||||
"1e-4",
|
||||
"--validation_frame_rate",
|
||||
"24",
|
||||
"--lr_scheduler",
|
||||
"constant_with_warmup",
|
||||
"--lr_warmup_steps",
|
||||
"100",
|
||||
"--lr_num_cycles",
|
||||
"1",
|
||||
"--beta1",
|
||||
"0.9",
|
||||
"--beta2",
|
||||
"0.95",
|
||||
"--weight_decay",
|
||||
"1e-4",
|
||||
"--epsilon",
|
||||
"1e-8",
|
||||
"--max_grad_norm",
|
||||
"1.0",
|
||||
"--validation_prompts",
|
||||
"A beautiful woman wearing a very revealing red lingerie thong and bra, walks through a modern, minimalist interior featuring a white sofa, black chairs, and a glass partition. Her confident and alluring demeanor is highlighted by the soft lighting and the room's neutral tones. As she strides forward, her bare feet touch the carpet, and her brown hair cascades over her shoulders. The setting includes a sleek white dining table, and black dining table chairs, a couch to the left, and a bookshelf filled with books and decorative items.@@@96x512x768",
|
||||
"--validation_pose_videos",
|
||||
"/home/ubuntu/upstream/finetrainers/video-dataset-pose-test/poses/stasyq-375-Scene-021-pose.mp4",
|
||||
"--validation_img_ref_videos",
|
||||
"/home/ubuntu/upstream/finetrainers/video-dataset-pose-test/videos/stasyq-375-Scene-021.mp4",
|
||||
"--num_validation_videos",
|
||||
"1",
|
||||
"--validation_steps",
|
||||
"500",
|
||||
"--tracker_name",
|
||||
"finetrainers-ltxv",
|
||||
"--output_dir",
|
||||
"ltx-video/ltxv_strip",
|
||||
"--nccl_timeout",
|
||||
"1800",
|
||||
"--report_to",
|
||||
"wandb",
|
||||
"--resume_from_checkpoint=latest"
|
||||
],
|
||||
"justMyCode": false
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user