From 78d0fd332ad49a67577bfe30c32f00ef24dabc17 Mon Sep 17 00:00:00 2001 From: Rahul Dubey Date: Wed, 29 Jan 2025 12:38:12 +0530 Subject: [PATCH] Update finetune_deepseekcoder.py Using torch.float16 or torch.cuda.amp can significantly reduce memory usage and speed up training by performing computations with lower precision. --- finetune/finetune_deepseekcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/finetune/finetune_deepseekcoder.py b/finetune/finetune_deepseekcoder.py index b96b849..1db452b 100644 --- a/finetune/finetune_deepseekcoder.py +++ b/finetune/finetune_deepseekcoder.py @@ -143,7 +143,7 @@ def train(): model = transformers.AutoModelForCausalLM.from_pretrained( model_args.model_name_or_path, - torch_dtype=torch.bfloat16 + torch_dtype=torch.float16 ) if training_args.local_rank == 0: