From cff51418c7dacca6cf7f3a8d983bae256ec236e6 Mon Sep 17 00:00:00 2001
From: jhqxxx <jhqxxx@163.com>
Date: Tue, 28 Oct 2025 14:29:12 +0800
Subject: [PATCH] fix vision data t dim not divisible by temporal_patch_size

---
 src/models/qwen2_5vl/processor.rs | 10 ++++++++++
 src/models/qwen3vl/processor.rs   | 11 +++++++++++
 tests/test_qwen3vl.rs             |  4 +++-
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/models/qwen2_5vl/processor.rs b/src/models/qwen2_5vl/processor.rs
index 7c2c05a..c9148d2 100644
--- a/src/models/qwen2_5vl/processor.rs
+++ b/src/models/qwen2_5vl/processor.rs
@@ -107,6 +107,16 @@ impl Qwen2_5VLProcessor {
     }
 
     pub fn process_vision_tensor(&self, img_tensor: &Tensor) -> Result<(Tensor, Tensor)> {
+        // Check that data have `num_frames` divisible by `temporal_patch_size`
+        // img_tensor: (t, c, h, w)
+        let t = img_tensor.dim(0)?;
+        let img_tensor = if t % self.vision_setting.temporal_patch_size != 0 {
+            let repeat_num = self.vision_setting.temporal_patch_size - t % self.vision_setting.temporal_patch_size;
+            let repeats = img_tensor.i(t-1)?.repeat((repeat_num, 1, 1, 1))?;
+            Tensor::cat(&[img_tensor, &repeats], 0)?
+        } else {
+            img_tensor.clone()
+        };
         let channel = img_tensor.dim(1)?;
         let grid_t = img_tensor.dim(0)? / self.vision_setting.temporal_patch_size;
         let grid_h = img_tensor.dim(2)? / self.vision_setting.patch_size;
diff --git a/src/models/qwen3vl/processor.rs b/src/models/qwen3vl/processor.rs
index ab426bc..6e2c767 100644
--- a/src/models/qwen3vl/processor.rs
+++ b/src/models/qwen3vl/processor.rs
@@ -158,6 +158,16 @@ impl Qwen3VLProcessor {
     }
 
     pub fn process_vision_tensor(&self, img_tensor: &Tensor) -> Result<(Tensor, Tensor)> {
+        // Check that data have `num_frames` divisible by `temporal_patch_size`
+        // img_tensor: (t, c, h, w)
+        let t = img_tensor.dim(0)?;
+        let img_tensor = if t % self.img_process_cfg.temporal_patch_size != 0 {
+            let repeat_num = self.img_process_cfg.temporal_patch_size - t % self.img_process_cfg.temporal_patch_size;
+            let repeats = img_tensor.i(t-1)?.repeat((repeat_num, 1, 1, 1))?;
+            Tensor::cat(&[img_tensor, &repeats], 0)?
+        } else {
+            img_tensor.clone()
+        };
         let channel = img_tensor.dim(1)?;
         let grid_t = img_tensor.dim(0)? / self.img_process_cfg.temporal_patch_size;
         let grid_h = img_tensor.dim(2)? / self.img_process_cfg.patch_size;
@@ -241,6 +251,7 @@ impl Qwen3VLProcessor {
                 .broadcast_sub(img_mean)?
                 .broadcast_div(img_std)?
                 .contiguous()?;
+            // t 
             let (video_tensor, video_grid_thw) = self.process_vision_tensor(&video_tensor)?;
             pixel_values_vec.push(video_tensor);
             vision_grid_thws_vec.push(video_grid_thw);
diff --git a/tests/test_qwen3vl.rs b/tests/test_qwen3vl.rs
index ee8414a..06ae5f7 100644
--- a/tests/test_qwen3vl.rs
+++ b/tests/test_qwen3vl.rs
@@ -22,7 +22,7 @@ fn qwen3vl_generate() -> Result<()> {
                         "type": "video",
                         "video_url": 
                         {
-                            "url": "./assets/video/video_test.mp4"
+                            "url": "https://www.w3schools.com/html/movie.mp4"
                         }
                     },                          
                     {
@@ -34,6 +34,8 @@ fn qwen3vl_generate() -> Result<()> {
         ]
     }
     "#;
+    // ./assets/video/video_test.mp4
+
     let mes: ChatCompletionParameters = serde_json::from_str(message)?;
     let i_start = Instant::now();
     let mut qwen3vl = Qwen3VLGenerateModel::init(model_path, None, None)?;