model test add tps

src/models/deepseek_ocr/generate.rs

@@ -127,9 +127,10 @@ impl GenerateModel for DeepseekOCRGenerateModel {
             images_seq_mask = None;
             images_spatial_crop_t = None;
         }
+        let num_token = generate.len() as u32;
         let res = self.tokenizer.token_decode(generate)?;
         self.deepseekocr_model.clear_kv_cache();
-        let response = build_completion_response(res, &self.model_name);
+        let response = build_completion_response(res, &self.model_name, Some(num_token));
         Ok(response)
     }
 

src/models/hunyuan_ocr/generate.rs

@@ -119,9 +119,10 @@ impl<'a> GenerateModel for HunyuanOCRGenerateModel<'a> {
             pixel_values = None;
             image_grid_thw = None;
         }
+        let num_token = generate.len() as u32;
         let res = self.tokenizer.token_decode(generate)?;
         self.hunyuan_vl.clear_kv_cache();
-        let response = build_completion_response(res, &self.model_name);
+        let response = build_completion_response(res, &self.model_name, Some(num_token));
         Ok(response)
     }
 

src/models/minicpm4/generate.rs

@@ -78,9 +78,10 @@ impl<'a> GenerateModel for MiniCPMGenerateModel<'a> {
             seq_len = 1;
             input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
         }
+        let num_token = generate.len() as u32;
         let res = self.tokenizer.token_decode(generate)?;
         self.minicpm.clear_kv_cache();
-        let response = build_completion_response(res, &self.model_name);
+        let response = build_completion_response(res, &self.model_name, Some(num_token));
         Ok(response)
     }
     fn generate_stream(

src/models/paddleocr_vl/generate.rs

@@ -104,9 +104,10 @@ impl<'a> GenerateModel for PaddleOCRVLGenerateModel<'a> {
             pixel_values = None;
             image_grid_thw = None;
         }
+        let num_token = generate.len() as u32;
         let res = self.tokenizer.token_decode(generate)?;
         self.paddleocr_vl.clear_kv_cache();
-        let response = build_completion_response(res, &self.model_name);
+        let response = build_completion_response(res, &self.model_name, Some(num_token));
         Ok(response)
     }
 

src/models/qwen2_5vl/generate.rs

@@ -118,9 +118,10 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
             pixel_values = None;
             pixel_values_video = None;
         }
+        let num_token = generate.len() as u32;
         let res = self.tokenizer.token_decode(generate)?;
         self.qwen2_5_vl.clear_kv_cache();
-        let response = build_completion_response(res, &self.model_name);
+        let response = build_completion_response(res, &self.model_name, Some(num_token));
         Ok(response)
     }
 
@@ -167,6 +168,8 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
             let image_grid_thw = image_grid_thw.as_ref();
             let mut pixel_values_video = pixel_values_video.as_ref();
             let video_grid_thw = video_grid_thw.as_ref();
+            let mut tool_call_id = None;
+            let mut tool_call_content = String::new();
             for _ in 0..sample_len {
                 let logits = self.qwen2_5_vl.forward(
                     &input_ids,
@@ -203,8 +206,56 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
                     continue;
                 }
                 error_tokens.clear();
-                let chunk = build_completion_chunk_response(decoded_token, &self.model_name, None, None);
-                yield Ok(chunk);
+                // 处理特殊标记和工具调用
+                match decoded_token.as_str() {
+                    "<tool_call>" => {
+                        // 开始工具调用
+                        tool_call_id = Some(uuid::Uuid::new_v4().to_string());
+                        seqlen_offset += seq_len;
+                        seq_len = 1;
+                        input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+                        cache_position = Tensor::from_vec(vec![seqlen_offset as u32], 1, &self.device)?;
+                        pixel_values = None;
+                        pixel_values_video = None;
+                        continue;
+                    }
+                    "</tool_call>" => {
+                        // 结束工具调用
+                        let chunk = build_completion_chunk_response(
+                            decoded_token, 
+                            &self.model_name, 
+                            tool_call_id.clone(), 
+                            Some(tool_call_content.clone())
+                        );
+                        tool_call_id = None;
+                        tool_call_content = String::new();
+                        yield Ok(chunk);
+                    }
+                    _ => {
+                        if tool_call_id.is_some() {
+                            // 在工具调用过程中，收集工具调用内容
+                            tool_call_content.push_str(&decoded_token);
+                            seqlen_offset += seq_len;
+                            seq_len = 1;
+                            input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+                            cache_position = Tensor::from_vec(vec![seqlen_offset as u32], 1, &self.device)?;
+                            pixel_values = None;
+                            pixel_values_video = None;
+                            continue;
+                        } else {
+                            // 正常文本输出
+                            let chunk = build_completion_chunk_response(
+                                decoded_token, 
+                                &self.model_name, 
+                                None, 
+                                None
+                            );
+                            yield Ok(chunk);
+                        }
+                    }
+                }          
+                // let chunk = build_completion_chunk_response(decoded_token, &self.model_name, None, None);
+                // yield Ok(chunk);
                 if next_token == self.endoftext_id || next_token == self.im_end_id {
                     break;
                 }

src/models/qwen2_5vl/processor.rs

@@ -72,6 +72,9 @@ impl Qwen2_5VLProcessor {
                     if let ChatMessageContentPart::Image(img_part) = part {
                         let img_url = img_part.image_url;
                         vision_map.get_mut("image").unwrap().push(img_url.url);
+                    // } else if let ChatMessageContentPart::Video(video_part) = part {
+                    //     let video_url = video_part.video_url;
+                    //     vision_map.get_mut("video").unwrap().push(video_url.url);
                     }
                 }
             }

src/models/qwen3vl/generate.rs

@@ -120,9 +120,10 @@ impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> {
             pixel_values = None;
             pixel_values_video = None;
         }
+        let num_token = generate.len() as u32;
         let res = self.tokenizer.token_decode(generate)?;
         self.qwen3_vl.clear_kv_cache();
-        let response = build_completion_response(res, &self.model_name);
+        let response = build_completion_response(res, &self.model_name, Some(num_token));
         Ok(response)
     }
 
@@ -174,18 +175,18 @@ impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> {
             let mut tool_call_id = None;
             let mut tool_call_content = String::new();
             for _ in 0..sample_len {
-            let logits = self.qwen3_vl.forward(
-                &input_ids,
-                pixel_values,
-                image_grid_thw,
-                pixel_values_video,
-                video_grid_thw,
-                Some(&cache_position),
-                seqlen_offset,
-            )?;
-            let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
-            let next_token = logit_processor.sample(&logits)?;
-            let mut decode_ids = Vec::new();
+                let logits = self.qwen3_vl.forward(
+                    &input_ids,
+                    pixel_values,
+                    image_grid_thw,
+                    pixel_values_video,
+                    video_grid_thw,
+                    Some(&cache_position),
+                    seqlen_offset,
+                )?;
+                let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
+                let next_token = logit_processor.sample(&logits)?;
+                let mut decode_ids = Vec::new();
                 if !error_tokens.is_empty() {
                     decode_ids.extend_from_slice(&error_tokens);
                 }
@@ -206,27 +207,54 @@ impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> {
                 }
                 error_tokens.clear();
 
-                if decoded_token.as_str() == "<tool_call>" {
-                    tool_call_id = Some(uuid::Uuid::new_v4().to_string());
-                    continue;
-                } else {
-                    if decoded_token.as_str() == "</tool_call>" {
-                        let chunk = build_completion_chunk_response(decoded_token, &self.model_name, tool_call_id.clone(), Some(tool_call_content.clone()));
+                // 处理特殊标记和工具调用
+                match decoded_token.as_str() {
+                    "<tool_call>" => {
+                        // 开始工具调用
+                        tool_call_id = Some(uuid::Uuid::new_v4().to_string());
+                        seqlen_offset += seq_len;
+                        seq_len = 1;
+                        input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+                        cache_position = Tensor::from_vec(vec![seqlen_offset as u32], 1, &self.device)?;
+                        pixel_values = None;
+                        pixel_values_video = None;
+                        continue;
+                    }
+                    "</tool_call>" => {
+                        // 结束工具调用
+                        let chunk = build_completion_chunk_response(
+                            decoded_token,
+                            &self.model_name,
+                            tool_call_id.clone(),
+                            Some(tool_call_content.clone())
+                        );
                         tool_call_id = None;
                         tool_call_content = String::new();
                         yield Ok(chunk);
                     }
-                    else {
+                    _ => {
                         if tool_call_id.is_some() {
+                            // 在工具调用过程中，收集工具调用内容
                             tool_call_content.push_str(&decoded_token);
+                            seqlen_offset += seq_len;
+                            seq_len = 1;
+                            input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+                            cache_position = Tensor::from_vec(vec![seqlen_offset as u32], 1, &self.device)?;
+                            pixel_values = None;
+                            pixel_values_video = None;
                             continue;
-                        }
-                        else {
-                            let chunk = build_completion_chunk_response(decoded_token, &self.model_name, None, None);
+                        } else {
+                            // 正常文本输出
+                            let chunk = build_completion_chunk_response(
+                                decoded_token,
+                                &self.model_name,
+                                None,
+                                None
+                            );
                             yield Ok(chunk);
                         }
                     }
-                }                
+                }
                 if next_token == self.eos_token_id1 || next_token == self.eos_token_id2 {
                     break;
                 }

src/models/qwen3vl/model.rs

@@ -730,7 +730,6 @@ impl Qwen3VLTextModel {
         deepstack_visual_embeds: Option<Vec<Tensor>>,
     ) -> Result<Tensor> {
         let (b_size, seq_len, _) = inputs_embeds.dims3()?;
-
         let position_ids = match position_ids {
             Some(ids) => ids.clone(),
             None => Tensor::arange(