storytold/FineTrainers-Conditioning
2
0
Fork 0
mirror of https://github.com/storytold/FineTrainers-Conditioning.git synced 2026-10-17 00:23:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

RoPE fixes for 1.5, bfloat16 support in prepare_dataset, gradient_accumulation grad norm undefined fix (#107)

Browse Source 
* rope scaling fixes for cog 1.5

* grad norm log fix

* bf16 prepare dataset fix

* update

* style
This commit is contained in:
Aryan
2024-12-02 15:12:12 +05:30
committed by GitHub
parent 76a9f2b911
commit 6c41984d8a
5 changed files with 58 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
training/utils.py
+4 -2
View File
@@ -208,9 +208,12 @@ def prepare_rotary_positional_embeddings(
grid_width = width // (vae_scale_factor_spatial * patch_size)
base_size_width = base_width // (vae_scale_factor_spatial * patch_size)
base_size_height = base_height // (vae_scale_factor_spatial * patch_size)
if patch_size_t is None:
# CogVideoX 1.0
grid_crops_coords = get_resize_crop_region_for_grid((grid_height, grid_width), base_size_width, base_size_height)
grid_crops_coords = get_resize_crop_region_for_grid(
(grid_height, grid_width), base_size_width, base_size_height
)
freqs_cos, freqs_sin = get_3d_rotary_pos_embed(
embed_dim=attention_head_dim,
crops_coords=grid_crops_coords,
@@ -230,7 +233,6 @@ def prepare_rotary_positional_embeddings(
max_size=(base_size_height, base_size_width),
)
freqs_cos = freqs_cos.to(device=device)
freqs_sin = freqs_sin.to(device=device)
return freqs_cos, freqs_sin