From 4cd5a8e53b76523544dafc221ae58bf79f7f0fa0 Mon Sep 17 00:00:00 2001 From: CrossProduct Date: Tue, 7 Jan 2025 20:41:12 +0000 Subject: [PATCH] Used disable grad for component on lora fine tuning enabled --- finetrainers/trainer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/finetrainers/trainer.py b/finetrainers/trainer.py index 6716f78..e948411 100644 --- a/finetrainers/trainer.py +++ b/finetrainers/trainer.py @@ -390,7 +390,8 @@ class Trainer: self._enable_grad_for_components(components=[self.transformer]) else: logger.info("Lora Fine Tuning Enabled") - + self._disable_grad_for_components(components=[self.transformer]) + # For mixed precision training we cast all non-trainable weights (vae, text_encoder and transformer) to half-precision # as these weights are only used for inference, keeping weights in full precision is not required. weight_dtype = self._get_training_dtype(accelerator=self.state.accelerator)