add qwen3vl 4b, 8b, 32b

README.md

@@ -25,7 +25,7 @@
 * ubuntu/WSL
 ```bash
 sudo apt-get update
-sudo apt-get install -y pkg-config ffmpeg libavutil-dev libavcodec-dev libavformat-dev libavfilter-dev libavdevice-dev libswresample-dev libswscale-dev
+sudo apt-get install -y clang pkg-config ffmpeg libavutil-dev libavcodec-dev libavformat-dev libavfilter-dev libavdevice-dev libswresample-dev libswscale-dev
 ```
 * windows参考： https://github.com/zmwangx/rust-ffmpeg/wiki/Notes-on-building
 

src/main.rs

@@ -78,7 +78,11 @@ async fn main() -> anyhow::Result<()> {
     let model_id = match args.model {
         WhichModel::MiniCPM4_0_5B => "OpenBMB/MiniCPM4-0.5B",
         WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct", 
+        WhichModel::Qwen2_5vl7B => "Qwen/Qwen2.5-VL-7B-Instruct",
         WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct", 
+        WhichModel::Qwen3vl4B => "Qwen/Qwen3-VL-4B-Instruct", 
+        WhichModel::Qwen3vl8B => "Qwen/Qwen3-VL-8B-Instruct", 
+        WhichModel::Qwen3vl32B => "Qwen/Qwen3-VL-32B-Instruct"
     };
     let model_path = match args.weight_path {
         Some(path) => path,

src/models/mod.rs

@@ -21,8 +21,16 @@ pub enum WhichModel {
     MiniCPM4_0_5B,
     #[value(name = "qwen2.5vl-3b")]
     Qwen2_5vl3B,
+    #[value(name = "qwen2.5vl-7b")]
+    Qwen2_5vl7B,
     #[value(name = "qwen3vl-2b")]
     Qwen3vl2B,
+    #[value(name = "qwen3vl-4b")]
+    Qwen3vl4B,
+    #[value(name = "qwen3vl-8b")]
+    Qwen3vl8B,
+    #[value(name = "qwen3vl-32b")]
+    Qwen3vl32B,
 }
 
 pub trait GenerateModel {
@@ -84,10 +92,26 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_
             let model = Qwen2_5VLGenerateModel::init(path, None, None)?;
             ModelInstance::Qwen2_5VL(model)
         }
+        WhichModel::Qwen2_5vl7B => {
+            let model = Qwen2_5VLGenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen2_5VL(model)
+        }
         WhichModel::Qwen3vl2B => {
             let model = Qwen3VLGenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3VL(model)
         } 
+        WhichModel::Qwen3vl4B => {
+            let model = Qwen3VLGenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3VL(model)
+        } 
+        WhichModel::Qwen3vl8B => {
+            let model = Qwen3VLGenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3VL(model)
+        } 
+        WhichModel::Qwen3vl32B => {
+            let model = Qwen3VLGenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3VL(model)
+        }
     };
     Ok(model)
 }

src/models/qwen3vl/config.rs

@@ -42,7 +42,6 @@ pub struct Qwen3VLTextConfig {
     pub rms_norm_eps: f64,
     pub rope_scaling: RopeScaling,
     pub rope_theta: f32,
-    pub tie_word_embeddings: bool,
     pub use_cache: bool,
     pub vocab_size: usize,
 }

src/models/qwen3vl/generate.rs

@@ -47,7 +47,6 @@ impl<'a> Qwen3VLGenerateModel<'a> {
         let pre_processor = Qwen3VLProcessor::new(path, &device, dtype)?;
         let model_list = find_type_files(path, "safetensors")?;
         let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, &device)? };
-        let vb = vb.pp("model");
         let qwen3_vl = Qwen3VLModel::new(cfg, vb)?;
         let generation_config_path = path.to_string() + "/generation_config.json";
         let generation_config: Qwen3VLGenerationConfig =

src/models/qwen3vl/model.rs

@@ -568,7 +568,7 @@ impl Qwen3VLTextAttention {
     pub fn new(config: Qwen3VLTextConfig, vb: VarBuilder) -> Result<Self> {
         let hidden_size = config.hidden_size;
         let num_attention_heads = config.num_attention_heads;
-        let head_dim = hidden_size / num_attention_heads;
+        let head_dim = config.head_dim;
         let num_key_value_heads = config.num_key_value_heads;
         let num_kv_groups = num_attention_heads / num_key_value_heads;
         let scaling = 1f64 / f64::sqrt(head_dim as f64);
@@ -576,7 +576,7 @@ impl Qwen3VLTextAttention {
             let q_proj = linear(hidden_size, num_attention_heads * head_dim, vb.pp("q_proj"))?;
             let k_proj = linear(hidden_size, num_key_value_heads * head_dim, vb.pp("k_proj"))?;
             let v_proj = linear(hidden_size, num_key_value_heads * head_dim, vb.pp("v_proj"))?;
-            let o_proj = linear(hidden_size, hidden_size, vb.pp("o_proj"))?;
+            let o_proj = linear(num_attention_heads * head_dim, hidden_size, vb.pp("o_proj"))?;
             (q_proj, k_proj, v_proj, o_proj)
         } else {
             let q_proj =
@@ -585,7 +585,7 @@ impl Qwen3VLTextAttention {
                 linear_no_bias(hidden_size, num_key_value_heads * head_dim, vb.pp("k_proj"))?;
             let v_proj =
                 linear_no_bias(hidden_size, num_key_value_heads * head_dim, vb.pp("v_proj"))?;
-            let o_proj = linear_no_bias(hidden_size, hidden_size, vb.pp("o_proj"))?;
+            let o_proj = linear_no_bias(num_attention_heads * head_dim, hidden_size, vb.pp("o_proj"))?;
             (q_proj, k_proj, v_proj, o_proj)
         };
         let q_norm = rms_norm(head_dim, config.rms_norm_eps, vb.pp("q_norm"))?;
@@ -652,7 +652,7 @@ impl Qwen3VLTextAttention {
             attention_mask,
             self.scaling,
         )?;
-        let attn_output = attn_output.reshape((b_sz, q_len, self.hidden_size))?;
+        let attn_output = attn_output.reshape((b_sz, q_len, self.num_attention_heads*self.head_dim))?;
         let attn_output = attn_output.apply(&self.o_proj)?;
         Ok(attn_output)
     }
@@ -738,7 +738,7 @@ impl Qwen3VLTextModel {
             layers.push(layer)
         }
         let norm = rms_norm(config.hidden_size, config.rms_norm_eps, vb.pp("norm"))?;
-        let head_dim = config.hidden_size / config.num_attention_heads;
+        let head_dim = config.head_dim;
         let rotary_emb = Qwen3VLTextRotaryEmbedding::new(head_dim, config.rope_theta);
         let mrope_section = config.rope_scaling.mrope_section.clone();
         Ok(Self {
@@ -823,10 +823,11 @@ pub struct Qwen3VLModel {
 
 impl Qwen3VLModel {
     pub fn new(config: Qwen3VLConfig, vb: VarBuilder) -> Result<Self> {
+        let vb_m = vb.pp("model");
         let config = config.clone();
-        let visual = Qwen3VLVisionModel::new(config.vision_config.clone(), vb.pp("visual"))?;
+        let visual = Qwen3VLVisionModel::new(config.vision_config.clone(), vb_m.pp("visual"))?;
         let language_model =
-            Qwen3VLTextModel::new(config.text_config.clone(), vb.pp("language_model"))?;
+            Qwen3VLTextModel::new(config.text_config.clone(), vb_m.pp("language_model"))?;
         let lm_head = if config.tie_word_embeddings {
             Linear::new(language_model.embed_tokens.embeddings().clone(), None)
         } else {

src/utils/mod.rs

@@ -21,7 +21,7 @@ pub fn get_device(device: Option<&Device>) -> Device {
         None => {
             #[cfg(feature = "cuda")]
             {
-                Device::new_cuda(0).unwrap_or(Device::Cpu)
+                Device::new_cuda(6).unwrap_or(Device::Cpu)
             }
             #[cfg(not(feature = "cuda"))]
             {

src/utils/tensor_utils.rs

@@ -8,10 +8,15 @@ pub fn prepare_causal_attention_mask(
     device: &Device,
 ) -> Result<Tensor> {
     // Sliding window mask?
-    let mask: Vec<_> = (0..tgt_len)
+    // let mask: Vec<f32> = (0..tgt_len)
-        .flat_map(|i| (0..tgt_len).map(move |j| if i < j { f32::NEG_INFINITY } else { 0. }))
+    //     .flat_map(|i| (0..tgt_len).map(move |j| if i < j { f32::NEG_INFINITY } else { 0. }))
-        .collect();
+    //     .collect();
-    let mask = Tensor::from_slice(&mask, (tgt_len, tgt_len), device)?;
+    // let mask = Tensor::from_vec(mask, (tgt_len, tgt_len), device)?;
+    let arange = Tensor::arange(0u32, tgt_len as u32, device)?;
+    let arange = arange.unsqueeze(1)?.broadcast_as((tgt_len, tgt_len))?;
+    let upper_triangle = arange.t()?.gt(&arange)?;
+    let mask = upper_triangle.where_cond(&Tensor::new(f32::NEG_INFINITY, device)?.broadcast_as(arange.shape())?, &Tensor::new(0f32, device)?.broadcast_as(arange.shape())?)?;
+
     let mask = if seqlen_offset > 0 {
         let mask0 = Tensor::zeros((tgt_len, seqlen_offset), DType::F32, device)?;
         Tensor::cat(&[&mask0, &mask], D::Minus1)?