DeepSeek/DeepSeek-V3
1
0
Fork 0
mirror of https://github.com/deepseek-ai/DeepSeek-V3 synced 2025-06-26 18:17:55 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Release DeepSeek-V3

Browse Source
This commit is contained in:
stack-heap-overflow
2024-12-26 19:01:57 +08:00
parent 4b58dc6bfc
commit 4c2fdb8f55
17 changed files with 1567 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
inference/configs/config_16B.json Normal file
View File

@@ -0,0 +1,19 @@
{
"vocab_size": 102400,
"dim": 2048,
"inter_dim": 10944,
"moe_inter_dim": 1408,
"n_layers": 27,
"n_dense_layers": 1,
"n_heads": 16,
"n_routed_experts": 64,
"n_shared_experts": 2,
"n_activated_experts": 6,
"route_scale": 1.0,
"q_lora_rank": 0,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"mscale": 0.707
}