Add DeepseekV3 AutoTP. (#7045)

Yejing-Lai · shenzheyu · commit 91d05e2bdf0c · 2025-03-05T17:55:44.000-05:00
Add deepseekv3 autotp.

Signed-off-by: Lai, Yejing &lt;yejing.lai@intel.com&gt;
Signed-off-by: Zheyu SHEN &lt;zyshen@umd.edu&gt;
diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py
@@ -137,7 +137,8 @@ def is_load_module(module):
             "LPLayerNorm", "SharedEmbedding", "OPTLearnedPositionalEmbedding", "LlamaRMSNorm", "FalconLinear",
             "MistralRMSNorm", "T5LayerNorm", "MixtralRMSNorm", "Phi3RotaryEmbedding", "Phi3SuScaledRotaryEmbedding",
             "Phi3RMSNorm", "YuanRMSNorm", "YuanRotaryEmbedding", "Phi3LongRoPEScaledRotaryEmbedding", "Qwen2RMSNorm",
-            "DeepseekV2RMSNorm", "DeepseekV2YarnRotaryEmbedding", "MoEGate"
+            "DeepseekV2RMSNorm", "DeepseekV3RMSNorm", "DeepseekV2YarnRotaryEmbedding", "DeepseekV3YarnRotaryEmbedding",
+            "MoEGate"
         ]
         return module.__class__ in load_layers or module._get_name() in load_layer_names
 

Original file line number	Diff line number	Diff line change
`@@ -137,7 +137,8 @@ def is_load_module(module):`
`137`	`137`	`"LPLayerNorm", "SharedEmbedding", "OPTLearnedPositionalEmbedding", "LlamaRMSNorm", "FalconLinear",`
`138`	`138`	`"MistralRMSNorm", "T5LayerNorm", "MixtralRMSNorm", "Phi3RotaryEmbedding", "Phi3SuScaledRotaryEmbedding",`
`139`	`139`	`"Phi3RMSNorm", "YuanRMSNorm", "YuanRotaryEmbedding", "Phi3LongRoPEScaledRotaryEmbedding", "Qwen2RMSNorm",`
`140`		`- "DeepseekV2RMSNorm", "DeepseekV2YarnRotaryEmbedding", "MoEGate"`
	`140`	`+ "DeepseekV2RMSNorm", "DeepseekV3RMSNorm", "DeepseekV2YarnRotaryEmbedding", "DeepseekV3YarnRotaryEmbedding",`
	`141`	`+ "MoEGate"`
`141`	`142`	`]`
`142`	`143`	`return module.__class__ in load_layers or module._get_name() in load_layer_names`
`143`	`144`