Merge branch 'InternLM:main' into turbomind-linear-gdn-prefix-caching

Author: lapy

lmdeploy/pytorch/engine/executor/ray_executor.py

@@ -489,7 +489,7 @@ async def forward_async(self, inputs):
             finally:
                 # free ray.put inputs
                 try:
-                    ray._private.internal_api.free(self._prev_inputs)
+                    ray.internal.free(self._prev_inputs, local_only=False)
                 except Exception as e:
                     logger.warning(f'Free input ref failed: {e}')
 

lmdeploy/pytorch/engine/model_agent/agent.py

@@ -1208,6 +1208,7 @@ async def sleep(self, level: int = 1):
         if self.dist_config.dp > 1:
             await self.state.to_sleep.wait()
         self.cache_engine = None
+        self.state_cache_engine = None
         self.reset_graph_runner()
         device = 'cpu' if level == 1 else 'meta'
         self.patched_model.get_model().to(device=device, non_blocking=True)
@@ -1245,4 +1246,5 @@ def release(self):
         self.reset_graph_runner()
         self.patched_model = None
         self.cache_engine = None
+        self.state_cache_engine = None
         torch.cuda.empty_cache()

lmdeploy/pytorch/models/qwen3_vl_moe.py

@@ -10,7 +10,7 @@
 from lmdeploy.pytorch.model_inputs import StepContextManager
 from lmdeploy.pytorch.weight_loader.model_weight_loader import load_weight
 
-from .patch import add_prefix
+from .patch import add_prefix, get_build_model_context
 from .qwen3_moe import Qwen3MoeModel
 from .qwen3_vl import Qwen3VLForConditionalGeneration
 from .qwen3_vl import Qwen3VLTextRotaryEmbedding as Qwen3VLMoeTextRotaryEmbedding
@@ -44,6 +44,7 @@ def forward(
         # args for deepstack
         visual_pos_masks: torch.Tensor | None = None,
         deepstack_visual_embeds: list[torch.Tensor] | None = None,
+        all_routed_experts: torch.Tensor | None = None,
     ):
         """Rewrite of LlamaModel.forward."""
 
@@ -73,6 +74,7 @@ def forward(
                 past_key_value=past_key_value,
                 residual=residual,
                 attn_metadata=attn_metadata,
+                all_routed_experts=all_routed_experts,
             )
 
             # add visual features to the hidden states of first several layers
@@ -129,6 +131,75 @@ def __init__(
                                                   dtype=dtype,
                                                   device=device,
                                                   prefix=add_prefix('language_model', prefix))
+        # for router replay
+        bm_ctx = get_build_model_context()
+        self.enable_return_routed_experts = bm_ctx.enable_return_routed_experts
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        position_ids: torch.Tensor,
+        past_key_values: list[list[torch.Tensor]],
+        attn_metadata: Any = None,
+        inputs_embeds: torch.Tensor = None,
+        mrope_position_ids: torch.Tensor = None,
+        pixel_values: torch.Tensor = None,
+        vis_cu_seqlens: torch.Tensor = None,
+        vis_pos_emb: torch.Tensor = None,
+        image_mask: torch.Tensor = None,
+        pos_embeds: torch.Tensor = None,
+        grid_thw: torch.Tensor = None,
+        **kwargs,
+    ):
+        """Model forward, return logits."""
+
+        visual_pos_masks = None
+        deepstack_visual_embeds = None
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+            if pixel_values is not None:
+                dtype = inputs_embeds.dtype
+                pixel_values = pixel_values.to(dtype)
+                vis_pos_emb = (vis_pos_emb[0].to(dtype), vis_pos_emb[1].to(dtype))
+
+                # get image embeds and deepstack visual embeds
+                image_embeds, deepstack_visual_embeds = self.visual(pixel_values,
+                                                                    cu_seqlens=vis_cu_seqlens,
+                                                                    rotary_pos_emb=vis_pos_emb,
+                                                                    pos_embeds=pos_embeds)
+
+                # split image embeds per sample
+                split_sizes = (grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()
+                image_embeds = torch.split(image_embeds, split_sizes)
+                image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, dtype)
+
+                # mask and scatter to create final input embeddings
+                expanded_image_mask = image_mask.unsqueeze(-1).expand_as(inputs_embeds)
+                inputs_embeds = inputs_embeds.masked_scatter(expanded_image_mask, image_embeds)
+
+                visual_pos_masks = expanded_image_mask
+
+        # router replay
+        all_routed_experts = None
+        if self.enable_return_routed_experts:
+            all_routed_experts = input_ids.new_empty((input_ids.size(1), self.config.text_config.num_hidden_layers,
+                                                      self.config.text_config.num_experts_per_tok),
+                                                     dtype=torch.uint16)
+        hidden_states = self.language_model(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            attn_metadata=attn_metadata,
+            inputs_embeds=inputs_embeds,
+            mrope_position_ids=mrope_position_ids,
+            # args for deepstack
+            visual_pos_masks=visual_pos_masks,
+            deepstack_visual_embeds=deepstack_visual_embeds,
+            all_routed_experts=all_routed_experts)
+        if all_routed_experts is None:
+            return hidden_states
+        return dict(hidden_states=hidden_states, all_routed_experts=all_routed_experts)
 
     def _load_weight_experts(self, name: str, loaded_weight: torch.Tensor, params_dict: dict[str, nn.Parameter],
                              expert_params_mapping: list):