storytold/FineTrainers-Conditioning
2
0
Fork 0
mirror of https://github.com/storytold/FineTrainers-Conditioning.git synced 2026-10-29 09:07:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

sft with multigpu

Browse Source
This commit is contained in:
zhipuch
2024-11-12 15:20:41 +08:00
parent d63a826f37
commit ac5f417555
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
training/cogvideox_text_to_video_sft.py
+1 -1
View File
@@ -743,7 +743,7 @@ def main(args):
last_lr = lr_scheduler.get_last_lr()[0] if lr_scheduler is not None else args.learning_rate last_lr = lr_scheduler.get_last_lr()[0] if lr_scheduler is not None else args.learning_rate
logs = {"loss": loss.detach().item(), "lr": last_lr} logs = {"loss": loss.detach().item(), "lr": last_lr}
# gradnorm + deepspeed: https://github.com/microsoft/DeepSpeed/issues/4555 # gradnorm + deepspeed: https://github.com/microsoft/DeepSpeed/issues/4555
if accelerator.distributed_type != DistributedType.DEEPSPEED: if accelerator.sync_gradients and accelerator.distributed_type != DistributedType.DEEPSPEED:
logs.update( logs.update(
{ {
"gradient_norm_before_clip": gradient_norm_before_clip, "gradient_norm_before_clip": gradient_norm_before_clip,