storytold/FineTrainers-Conditioning
2
0
Fork 0
mirror of https://github.com/storytold/FineTrainers-Conditioning.git synced 2026-10-24 08:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

torch.is_tensor check

Browse Source
This commit is contained in:
sayakpaul
2024-12-26 07:49:51 +05:30
parent 5757b1b886
commit 28ecfe6b65
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
finetrainers/trainer.py
+3 -1
View File
@@ -730,12 +730,14 @@ class Trainer:
if accelerator.distributed_type == DistributedType.DEEPSPEED: if accelerator.distributed_type == DistributedType.DEEPSPEED:
grad_norm = self.transformer.get_global_grad_norm() grad_norm = self.transformer.get_global_grad_norm()
# In some cases the grad norm may not return a float # In some cases the grad norm may not return a float
if hasattr(grad_norm, "item"): if torch.is_tensor(grad_norm):
grad_norm = grad_norm.item() grad_norm = grad_norm.item()
else: else:
grad_norm = accelerator.clip_grad_norm_( grad_norm = accelerator.clip_grad_norm_(
self.transformer.parameters(), self.args.max_grad_norm self.transformer.parameters(), self.args.max_grad_norm
) )
if torch.is_tensor(grad_norm):
grad_norm = grad_norm.item()
logs["grad_norm"] = grad_norm logs["grad_norm"] = grad_norm