storytold/FineTrainers-Conditioning
2
0
Fork 0
mirror of https://github.com/storytold/FineTrainers-Conditioning.git synced 2026-10-17 00:23:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

readme updates + refactor (#14)

Browse Source 
* refactor

* update

* update

* Update README.md

Co-authored-by: Sayak Paul <spsayakpaul@gmail.com>

* Update README.md

Co-authored-by: Sayak Paul <spsayakpaul@gmail.com>

* address review comments part i

* fix adapter name causing peft error

* prompts -> prompt

* update

* add video

* update

---------

Co-authored-by: Sayak Paul <spsayakpaul@gmail.com>
This commit is contained in:
Aryan
2024-10-10 18:03:45 +05:30
committed by GitHub
parent 2a70b16d3c
commit ee02219eac
11 changed files with 233 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
training/cogvideox_image_to_video_lora.py
+18 -29
View File
@@ -107,7 +107,7 @@ from diffusers import CogVideoXImageToVideoPipeline
from diffusers.utils import export_to_video, load_image
pipe = CogVideoXImageToVideoPipeline.from_pretrained("THUDM/CogVideoX-5b-I2V", torch_dtype=torch.bfloat16).to("cuda")
pipe.load_lora_weights("{repo_id}", weight_name="pytorch_lora_weights.safetensors", adapter_name=["cogvideox-lora"])
pipe.load_lora_weights("{repo_id}", weight_name="pytorch_lora_weights.safetensors", adapter_name="cogvideox-lora")
# The LoRA adapter weights are determined by what was used for training.
# In this case, we assume `--lora_alpha` is 32 and `--rank` is 64.
@@ -465,36 +465,25 @@ def main(args):
)
# Dataset and DataLoader
if not args.video_reshape_mode:
train_dataset = VideoDatasetWithResizing(
data_root=args.data_root,
dataset_file=args.dataset_file,
caption_column=args.caption_column,
video_column=args.video_column,
max_num_frames=args.max_num_frames,
id_token=args.id_token,
height_buckets=args.height_buckets,
width_buckets=args.width_buckets,
frame_buckets=args.frame_buckets,
load_tensors=args.load_tensors,
random_flip=args.random_flip,
image_to_video=True,
)
dataset_init_kwargs = {
"data_root": args.data_root,
"dataset_file": args.dataset_file,
"caption_column": args.caption_column,
"video_column": args.video_column,
"max_num_frames": args.max_num_frames,
"id_token": args.id_token,
"height_buckets": args.height_buckets,
"width_buckets": args.width_buckets,
"frame_buckets": args.frame_buckets,
"load_tensors": args.load_tensors,
"random_flip": args.random_flip,
"image_to_video": True,
}
if args.video_reshape_mode is None:
train_dataset = VideoDatasetWithResizing(**dataset_init_kwargs)
else:
train_dataset = VideoDatasetWithResizeAndRectangleCrop(
video_reshape_mode=args.video_reshape_mode,
data_root=args.data_root,
dataset_file=args.dataset_file,
caption_column=args.caption_column,
video_column=args.video_column,
max_num_frames=args.max_num_frames,
id_token=args.id_token,
height_buckets=args.height_buckets,
width_buckets=args.width_buckets,
frame_buckets=args.frame_buckets,
load_tensors=args.load_tensors,
random_flip=args.random_flip,
image_to_video=True,
video_reshape_mode=args.video_reshape_mode, **dataset_init_kwargs
)
def collate_fn(data):
training/cogvideox_text_to_video_lora.py
+18 -28
View File
@@ -103,10 +103,10 @@ Requires the [🧨 Diffusers library](https://github.com/huggingface/diffusers)
```py
import torch
from diffusers import CogVideoXPipeline
from diffusers import export_to_video
from diffusers.utils import export_to_video
pipe = CogVideoXPipeline.from_pretrained("THUDM/CogVideoX-5b", torch_dtype=torch.bfloat16).to("cuda")
pipe.load_lora_weights("{repo_id}", weight_name="pytorch_lora_weights.safetensors", adapter_name=["cogvideox-lora"])
pipe.load_lora_weights("{repo_id}", weight_name="pytorch_lora_weights.safetensors", adapter_name="cogvideox-lora")
# The LoRA adapter weights are determined by what was used for training.
# In this case, we assume `--lora_alpha` is 32 and `--rank` is 64.
@@ -462,34 +462,24 @@ def main(args):
)
# Dataset and DataLoader
if not args.video_reshape_mode:
train_dataset = VideoDatasetWithResizing(
data_root=args.data_root,
dataset_file=args.dataset_file,
caption_column=args.caption_column,
video_column=args.video_column,
max_num_frames=args.max_num_frames,
id_token=args.id_token,
height_buckets=args.height_buckets,
width_buckets=args.width_buckets,
frame_buckets=args.frame_buckets,
load_tensors=args.load_tensors,
random_flip=args.random_flip,
)
dataset_init_kwargs = {
"data_root": args.data_root,
"dataset_file": args.dataset_file,
"caption_column": args.caption_column,
"video_column": args.video_column,
"max_num_frames": args.max_num_frames,
"id_token": args.id_token,
"height_buckets": args.height_buckets,
"width_buckets": args.width_buckets,
"frame_buckets": args.frame_buckets,
"load_tensors": args.load_tensors,
"random_flip": args.random_flip,
}
if args.video_reshape_mode is None:
train_dataset = VideoDatasetWithResizing(**dataset_init_kwargs)
else:
train_dataset = VideoDatasetWithResizeAndRectangleCrop(
video_reshape_mode=args.video_reshape_mode,
data_root=args.data_root,
dataset_file=args.dataset_file,
caption_column=args.caption_column,
video_column=args.video_column,
max_num_frames=args.max_num_frames,
id_token=args.id_token,
height_buckets=args.height_buckets,
width_buckets=args.width_buckets,
frame_buckets=args.frame_buckets,
load_tensors=args.load_tensors,
random_flip=args.random_flip,
video_reshape_mode=args.video_reshape_mode, **dataset_init_kwargs
)
def collate_fn(data):
training/cogvideox_text_to_video_sft.py
+16 -26
View File
@@ -426,34 +426,24 @@ def main(args):
)
# Dataset and DataLoader
if not args.video_reshape_mode:
train_dataset = VideoDatasetWithResizing(
data_root=args.data_root,
dataset_file=args.dataset_file,
caption_column=args.caption_column,
video_column=args.video_column,
max_num_frames=args.max_num_frames,
id_token=args.id_token,
height_buckets=args.height_buckets,
width_buckets=args.width_buckets,
frame_buckets=args.frame_buckets,
load_tensors=args.load_tensors,
random_flip=args.random_flip,
)
dataset_init_kwargs = {
"data_root": args.data_root,
"dataset_file": args.dataset_file,
"caption_column": args.caption_column,
"video_column": args.video_column,
"max_num_frames": args.max_num_frames,
"id_token": args.id_token,
"height_buckets": args.height_buckets,
"width_buckets": args.width_buckets,
"frame_buckets": args.frame_buckets,
"load_tensors": args.load_tensors,
"random_flip": args.random_flip,
}
if args.video_reshape_mode is None:
train_dataset = VideoDatasetWithResizing(**dataset_init_kwargs)
else:
train_dataset = VideoDatasetWithResizeAndRectangleCrop(
video_reshape_mode=args.video_reshape_mode,
data_root=args.data_root,
dataset_file=args.dataset_file,
caption_column=args.caption_column,
video_column=args.video_column,
max_num_frames=args.max_num_frames,
id_token=args.id_token,
height_buckets=args.height_buckets,
width_buckets=args.width_buckets,
frame_buckets=args.frame_buckets,
load_tensors=args.load_tensors,
random_flip=args.random_flip,
video_reshape_mode=args.video_reshape_mode, **dataset_init_kwargs
)
def collate_fn(data):
training/prepare_dataset.py
+2 -2
View File
@@ -348,7 +348,7 @@ def save_videos(
for video_path in video_paths:
file.write(f"videos/{video_path.name}\n")
with open(output_dir.joinpath("prompts.txt").as_posix(), "a", encoding="utf-8") as file:
with open(output_dir.joinpath("prompt.txt").as_posix(), "a", encoding="utf-8") as file:
for prompt in prompts:
file.write(f"{prompt}\n")
@@ -398,7 +398,7 @@ def save_latents_and_embeddings(
for video_path in video_paths:
file.write(f"videos/{video_path.name}\n")
with open(output_dir.joinpath("prompts.txt").as_posix(), "a", encoding="utf-8") as file:
with open(output_dir.joinpath("prompt.txt").as_posix(), "a", encoding="utf-8") as file:
for prompt in prompts:
file.write(f"{prompt}\n")