From ac5f4175550036037f04924c988d367e71863598 Mon Sep 17 00:00:00 2001 From: zhipuch Date: Tue, 12 Nov 2024 15:20:41 +0800 Subject: [PATCH] sft with multigpu --- training/cogvideox_text_to_video_sft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/cogvideox_text_to_video_sft.py b/training/cogvideox_text_to_video_sft.py index 9b44217..d7a3586 100644 --- a/training/cogvideox_text_to_video_sft.py +++ b/training/cogvideox_text_to_video_sft.py @@ -743,7 +743,7 @@ def main(args): last_lr = lr_scheduler.get_last_lr()[0] if lr_scheduler is not None else args.learning_rate logs = {"loss": loss.detach().item(), "lr": last_lr} # gradnorm + deepspeed: https://github.com/microsoft/DeepSpeed/issues/4555 - if accelerator.distributed_type != DistributedType.DEEPSPEED: + if accelerator.sync_gradients and accelerator.distributed_type != DistributedType.DEEPSPEED: logs.update( { "gradient_norm_before_clip": gradient_norm_before_clip,