DeepSeek/DeepSeek-V3
1
0
Fork 0
mirror of https://github.com/deepseek-ai/DeepSeek-V3 synced 2025-06-26 18:17:55 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Release DeepSeek-V3

Browse Source
This commit is contained in:
stack-heap-overflow
2024-12-26 19:01:57 +08:00
parent 4b58dc6bfc
commit 4c2fdb8f55
17 changed files with 1567 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
inference/configs/config_16B.json Normal file
View File

@@ -0,0 +1,19 @@
{
"vocab_size": 102400,
"dim": 2048,
"inter_dim": 10944,
"moe_inter_dim": 1408,
"n_layers": 27,
"n_dense_layers": 1,
"n_heads": 16,
"n_routed_experts": 64,
"n_shared_experts": 2,
"n_activated_experts": 6,
"route_scale": 1.0,
"q_lora_rank": 0,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"mscale": 0.707
}

20
inference/configs/config_236B.json Normal file
View File

@@ -0,0 +1,20 @@
{
"vocab_size": 102400,
"dim": 5120,
"inter_dim": 12288,
"moe_inter_dim": 1536,
"n_layers": 60,
"n_dense_layers": 1,
"n_heads": 128,
"n_routed_experts": 160,
"n_shared_experts": 2,
"n_activated_experts": 6,
"n_expert_groups": 8,
"n_limited_groups": 3,
"route_scale": 16.0,
"q_lora_rank": 1536,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128
}

22
inference/configs/config_671B.json Normal file
View File

@@ -0,0 +1,22 @@
{
"vocab_size": 129280,
"dim": 7168,
"inter_dim": 18432,
"moe_inter_dim": 2048,
"n_layers": 61,
"n_dense_layers": 3,
"n_heads": 128,
"n_routed_experts": 256,
"n_shared_experts": 1,
"n_activated_experts": 8,
"n_expert_groups": 8,
"n_limited_groups": 4,
"route_scale": 2.5,
"score_func": "sigmoid",
"q_lora_rank": 1536,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"dtype": "fp8"
}