DeepSeek/ESFT
1
0
Fork 0
mirror of https://github.com/deepseek-ai/ESFT synced 2025-06-26 18:15:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

streamline code; add intermediate saving support for ep

Browse Source
This commit is contained in:
ZihanWang314
2025-05-22 07:21:52 +00:00
parent 98fd21ce21
commit f38f67706c
123 changed files with 710 additions and 5601 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
configs/base.yaml
View File

@@ -8,7 +8,7 @@ n_device: 8 # Number of devices
# Training settings
optim: adamw_torch_fused
steps: 500 # Number of training steps
steps: 1 # Number of training steps
learning_rate: 0.00001 # Learning rate
weight_decay: 0.1 # Weight decay for optimizer
warmup_steps: 0 # Number of warmup steps for learning rate scheduler
@@ -19,13 +19,13 @@ random_concat_ratio: 0.2 # Ratio of random concatenation
# Evaluation settings
eval_steps: 100 # Evaluate every X steps
save_steps: 100 # Save model every X steps
eval_steps: 1 # Evaluate every X steps
save_steps: 1 # Save model every X steps
# Tokenizer settings
# Additional settings (if needed)
gradient_checkpointing: true
gradient_accumulation_steps: 16 # Number of updates steps to accumulate before performing a backward/update pass
gradient_accumulation_steps: 4 # Number of updates steps to accumulate before performing a backward/update pass
max_grad_norm: 1.0 # Max gradient norm for gradient clipping
ep_size: 2