Fix model config (#4170)

* fix config

* fix moe

Author: CUHKSZzxy

lmdeploy/pytorch/config.py

@@ -29,9 +29,9 @@ def _update_torch_dtype(config: 'ModelConfig', dtype: str):
         config.dtype = torch.float16
         return config
 
-    torch_dtype = getattr(config.llm_config, 'dtype', None)
+    torch_dtype = getattr(config.hf_config, 'dtype', None)
     if torch_dtype is None:
-        torch_dtype = getattr(config.llm_config, 'torch_dtype', None)
+        torch_dtype = getattr(config.hf_config, 'torch_dtype', None)
 
     # deal with case when torch_dtype is not string but torch.dtype
     if isinstance(torch_dtype, torch.dtype):

lmdeploy/pytorch/configurations/default.py

@@ -14,13 +14,6 @@ def condition(cls, hf_config):
     @classmethod
     def build(cls, hf_config, model_path: str = None, **kwargs):
         """build."""
-
-        # for multi-modal models, get the language model config to build model config
-        if hasattr(hf_config, 'text_config'):
-            hf_config = hf_config.text_config
-        elif hasattr(hf_config, 'llm_config'):
-            hf_config = hf_config.llm_config
-
         head_dim = getattr(hf_config, 'head_dim', None)
         head_dim = head_dim or hf_config.hidden_size // hf_config.num_attention_heads
 

lmdeploy/pytorch/configurations/qwen3_vl.py

@@ -0,0 +1,21 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .builder import AutoModelConfigBuilder
+from .default import DefaultModelConfigBuilder
+
+
+class Qwen3VLModelConfigBuilder(AutoModelConfigBuilder):
+
+    @classmethod
+    def condition(cls, hf_config):
+        """config."""
+        return hf_config.model_type == 'qwen3_vl' or hf_config.model_type == 'qwen3_vl_moe'
+
+    @classmethod
+    def build(cls, hf_config, model_path: str = None, **kwargs):
+        """build."""
+        if hasattr(hf_config, 'quantization_config') and not hasattr(hf_config.text_config, 'quantization_config'):
+            setattr(hf_config.text_config, 'quantization_config', hf_config.quantization_config)
+        cfg = DefaultModelConfigBuilder.build(hf_config.text_config, model_path, **kwargs)
+        setattr(hf_config, 'dtype', hf_config.text_config.dtype)
+        cfg.hf_config = hf_config
+        return cfg