add Qwen3.5 mmproj gguf

README.md

@@ -25,6 +25,9 @@
 aha is a high-performance, cross-platform AI inference engine built with Rust and the Candle framework. It brings state-of-the-art AI models to your local machine—no API keys, no cloud dependencies, just pure, fast AI running directly on your hardware.
 
 ## Changelog
+### 2026-03-16
+- Added Qwen3.5 mmproj
+
 ### 2026-03-14
 - update rust version 
 - Added Qwen3.5 gguf support, but the 4B model still has issues; to be resolved.

README.zh-CN.md

@@ -25,6 +25,9 @@
 aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理引擎。将最先进的 AI 模型带到您的本地机器——无需 API 密钥，无需云依赖，纯粹、快速的 AI，直接在您的硬件上运行。
 
 ## 更新日志
+### 2026-03-16
+- 增加 Qwen3.5 mmproj
+
 ### 2026-03-14
 - 更新rust版本 
 - 增加了对 Qwen3.5 gguf 的支持，但 4B 模型仍然存在问题；待解决。

docs/changelog.md

@@ -5,6 +5,9 @@ All notable changes to aha will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+### 2026-03-16
+- Added Qwen3.5 mmproj
+
 ### 2026-03-14
 - update rust version 
 - Added Qwen3.5 gguf support, but the 4B model still has issues; to be resolved.

docs/changelog.zh-CN.md

@@ -5,6 +5,9 @@
 格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)，
 本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/spec/v2.0.0.html)。
 
+### 2026-03-16
+- 增加 Qwen3.5 mmproj
+
 ### 2026-03-14
 - 更新rust版本 
 - 增加了对 Qwen3.5 gguf 的支持，但 4B 模型仍然存在问题；待解决。

src/models/common/gguf.rs

@@ -3,13 +3,15 @@ use std::io::{Read, Seek};
 use ahash::AHashMap;
 use anyhow::{Result, anyhow};
 use candle_core::{
-    Device, Tensor,
+    DType, Device, Tensor,
     quantized::{
         QMatMul, QTensor,
         gguf_file::{self, Value},
     },
 };
-use candle_nn::{Conv1d, Conv1dConfig, Linear, Module, RmsNorm, VarBuilder, linear_b};
+use candle_nn::{
+    Activation, Conv1d, Conv1dConfig, LayerNorm, Linear, Module, RmsNorm, VarBuilder, linear_b,
+};
 use tokenizers::{self, AddedToken, Tokenizer, models::bpe::BPE};
 
 use crate::tokenizer::TokenizerModel;
@@ -37,12 +39,40 @@ impl<R: Read + Seek> Gguf<R> {
         Ok(QMatMul::from_qtensor(ws)?)
     }
 
+    pub fn quantize_linear(&mut self, prefix: &str, bias: bool) -> Result<QuantizedLinear> {
+        let weight = self.qmatmul(&format!("{prefix}.weight"))?;
+        let bias = if bias {
+            self.get_dequantized(&format!("{prefix}.bias")).ok()
+        } else {
+            None
+        };
+        Ok(QuantizedLinear::new(weight, bias))
+    }
+
     pub fn rms_norm(&mut self, name: &str, eps: f64) -> Result<RmsNorm> {
         let ws = self.ct.tensor(&mut self.reader, name, &self.device)?;
         let weight = ws.dequantize(&self.device)?;
         Ok(RmsNorm::new(weight, eps))
     }
 
+    pub fn layer_norm(&mut self, prefix: &str, eps: f64) -> Result<LayerNorm> {
+        let weight = self
+            .ct
+            .tensor(&mut self.reader, &format!("{prefix}.weight"), &self.device)?;
+        let weight = weight.dequantize(&self.device)?;
+        let bias = self
+            .ct
+            .tensor(&mut self.reader, &format!("{prefix}.bias"), &self.device);
+        let bias = match bias {
+            Ok(bias) => bias.dequantize(&self.device).ok(),
+            Err(_) => None,
+        };
+        match bias {
+            Some(bias) => Ok(LayerNorm::new(weight, bias, eps)),
+            None => Ok(LayerNorm::new_no_bias(weight, eps)),
+        }
+    }
+
     pub fn metadata(&self) -> &std::collections::HashMap<String, gguf_file::Value> {
         &self.ct.metadata
     }
@@ -55,6 +85,10 @@ impl<R: Read + Seek> Gguf<R> {
         Ok(self.tensor(name)?.dequantize(&self.device)?)
     }
 
+    pub fn get_dequantized_f16(&mut self, name: &str) -> Result<Tensor> {
+        Ok(self.tensor(name)?.dequantize_f16(&self.device)?)
+    }
+
     pub fn conv1d(
         &mut self,
         prefix: &str,
@@ -169,9 +203,36 @@ impl<R: Read + Seek> Gguf<R> {
     }
 }
 
+#[derive(Debug, Clone)]
+pub struct QuantizedLinear {
+    inner: QMatMul,
+    bias: Option<Tensor>,
+}
+
+impl QuantizedLinear {
+    pub fn new(inner: QMatMul, bias: Option<Tensor>) -> Self {
+        Self { inner, bias }
+    }
+}
+
+impl Module for QuantizedLinear {
+    fn forward(&self, xs: &Tensor) -> candle_core::Result<Tensor> {
+        let xs = if xs.dtype() == DType::F16 {
+            self.inner.forward_via_f16(xs)?
+        } else {
+            self.inner.forward(xs)?
+        };
+        if let Some(bias) = &self.bias {
+            xs.broadcast_add(&bias.to_dtype(xs.dtype())?)
+        } else {
+            Ok(xs)
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 pub enum ProjKind {
-    QuantizedProj(QMatMul),
+    QuantizedProj(QuantizedLinear),
     LinearProj(Linear),
 }
 
@@ -186,20 +247,52 @@ impl Module for ProjKind {
 
 #[derive(Debug, Clone)]
 pub struct GateUpDownMLPGguf {
-    gate_proj: ProjKind, // ffn_gate.weight
+    gate_proj: ProjKind,
-    up_proj: ProjKind,   // ffn_up.weight
+    up_proj: ProjKind,
-    down_proj: ProjKind, // ffn_down.weight
+    down_proj: ProjKind,
+    act: Activation,
 }
 
 impl GateUpDownMLPGguf {
-    pub fn new_from_gguf<R: Read + Seek>(gguf: &mut Gguf<R>, prefix: &str) -> Result<Self> {
+    pub fn new_from_gguf<R: Read + Seek>(
-        let gate_proj = gguf.qmatmul(&format!("{prefix}.ffn_gate.weight"))?;
+        gguf: &mut Gguf<R>,
-        let up_proj = gguf.qmatmul(&format!("{prefix}.ffn_up.weight"))?;
+        prefix: &str,
-        let down_proj = gguf.qmatmul(&format!("{prefix}.ffn_down.weight"))?;
+        bias: bool,
+        gate_name: Option<&str>,
+        up_name: Option<&str>,
+        down_name: Option<&str>,
+        act: Option<Activation>,
+    ) -> Result<Self> {
+        let gate_name = gate_name.unwrap_or("ffn_gate");
+        let up_name = up_name.unwrap_or("ffn_up");
+        let down_name = down_name.unwrap_or("ffn_down");
+        let gate_proj = gguf.qmatmul(&format!("{prefix}.{gate_name}.weight"))?;
+        let gate_bias = if bias {
+            gguf.get_dequantized(&format!("{prefix}.{gate_name}.bias"))
+                .ok()
+        } else {
+            None
+        };
+        let up_proj = gguf.qmatmul(&format!("{prefix}.{up_name}.weight"))?;
+        let up_bias = if bias {
+            gguf.get_dequantized(&format!("{prefix}.{up_name}.bias"))
+                .ok()
+        } else {
+            None
+        };
+        let down_proj = gguf.qmatmul(&format!("{prefix}.{down_name}.weight"))?;
+        let down_bias = if bias {
+            gguf.get_dequantized(&format!("{prefix}.{down_name}.bias"))
+                .ok()
+        } else {
+            None
+        };
+        let act = act.unwrap_or(Activation::Silu);
         Ok(Self {
-            gate_proj: ProjKind::QuantizedProj(gate_proj),
+            gate_proj: ProjKind::QuantizedProj(QuantizedLinear::new(gate_proj, gate_bias)),
-            up_proj: ProjKind::QuantizedProj(up_proj),
+            up_proj: ProjKind::QuantizedProj(QuantizedLinear::new(up_proj, up_bias)),
-            down_proj: ProjKind::QuantizedProj(down_proj),
+            down_proj: ProjKind::QuantizedProj(QuantizedLinear::new(down_proj, down_bias)),
+            act,
         })
     }
     pub fn new_from_vb(
@@ -210,6 +303,7 @@ impl GateUpDownMLPGguf {
         gate_pp_name: Option<&str>,
         up_pp_name: Option<&str>,
         down_pp_name: Option<&str>,
+        act: Option<Activation>,
     ) -> Result<Self> {
         let gate_pp_name = gate_pp_name.unwrap_or("gate_proj");
         let up_pp_name = up_pp_name.unwrap_or("up_proj");
@@ -217,18 +311,77 @@ impl GateUpDownMLPGguf {
         let gate_proj = linear_b(hidden_size, intermediate_size, bias, vb.pp(gate_pp_name))?;
         let up_proj = linear_b(hidden_size, intermediate_size, bias, vb.pp(up_pp_name))?;
         let down_proj = linear_b(intermediate_size, hidden_size, bias, vb.pp(down_pp_name))?;
+        let act = act.unwrap_or(Activation::Silu);
         Ok(Self {
             gate_proj: ProjKind::LinearProj(gate_proj),
             up_proj: ProjKind::LinearProj(up_proj),
             down_proj: ProjKind::LinearProj(down_proj),
+            act,
         })
     }
 }
 
 impl Module for GateUpDownMLPGguf {
     fn forward(&self, xs: &Tensor) -> candle_core::Result<Tensor> {
-        let w1 = self.gate_proj.forward(xs)?;
+        let w1 = self.gate_proj.forward(xs)?.apply(&self.act)?;
         let w3 = self.up_proj.forward(xs)?;
-        self.down_proj.forward(&(candle_nn::ops::silu(&w1)? * w3)?)
+        self.down_proj.forward(&(w1 * w3)?)
+    }
+}
+
+pub struct TwoLinearMLPGguf {
+    linear1: ProjKind,
+    linear2: ProjKind,
+    act: Activation,
+}
+
+impl TwoLinearMLPGguf {
+    pub fn new(
+        vb: VarBuilder,
+        // embedding_dim: usize,
+        // mlp_dim: usize,
+        in_dim: usize,
+        middle_dim: usize,
+        out_dim: usize,
+        act: Activation,
+        bias: bool,
+        linear1_pp_name: &str,
+        linear2_pp_name: &str,
+    ) -> Result<Self> {
+        let linear1 = linear_b(in_dim, middle_dim, bias, vb.pp(linear1_pp_name))?;
+        let linear2 = linear_b(middle_dim, out_dim, bias, vb.pp(linear2_pp_name))?;
+
+        Ok(Self {
+            linear1: ProjKind::LinearProj(linear1),
+            linear2: ProjKind::LinearProj(linear2),
+            act,
+        })
+    }
+
+    pub fn new_from_gguf<R: Read + Seek>(
+        gguf: &mut Gguf<R>,
+        prefix: &str,
+        bias: bool,
+        linear1_name: Option<&str>,
+        linear2_name: Option<&str>,
+        act: Option<Activation>,
+    ) -> Result<Self> {
+        let linear1_name = linear1_name.unwrap_or("ffn_up");
+        let linear2_name = linear2_name.unwrap_or("ffn_down");
+        let linear1 = gguf.quantize_linear(&format!("{prefix}.{linear1_name}"), bias)?;
+        let linear2 = gguf.quantize_linear(&format!("{prefix}.{linear2_name}"), bias)?;
+        let act = act.unwrap_or(Activation::Silu);
+        Ok(Self {
+            linear1: ProjKind::QuantizedProj(linear1),
+            linear2: ProjKind::QuantizedProj(linear2),
+            act,
+        })
+    }
+    pub fn forward(&self, xs: &Tensor) -> Result<Tensor> {
+        let xs = xs
+            .apply(&self.linear1)?
+            .apply(&self.act)?
+            .apply(&self.linear2)?;
+        Ok(xs)
     }
 }

src/models/common/mod.rs

@@ -1172,7 +1172,7 @@ pub fn conv1d_depthwise(input: &Tensor, weight: &Tensor, bias: Option<&Tensor>)
     // weight: (dim, 1, k) -> (dim, k)
     // input already padding
     let len_in = input.dim(2)?;
-    let weight = weight.squeeze(1)?;
+    let weight = weight.squeeze(1)?.to_dtype(input.dtype())?;
     let kernel_size = weight.dim(1)?;
     // len_out = (len_in - k + 2p) / s + 1, p = 0, s = 1
     let len_out = len_in - kernel_size + 1;
@@ -1189,7 +1189,7 @@ pub fn conv1d_depthwise(input: &Tensor, weight: &Tensor, bias: Option<&Tensor>)
         None => Ok(out),
         Some(bias) => {
             let b = bias.dims1()?;
-            let bias = bias.reshape((1, b, 1))?;
+            let bias = bias.reshape((1, b, 1))?.to_dtype(input.dtype())?;
             Ok(out.broadcast_add(&bias)?)
         }
     }

src/models/mod.rs

@@ -168,7 +168,7 @@ pub enum ModelInstance<'a> {
     Qwen3(Qwen3GenerateModel<'a>),
     Qwen3_5(Qwen3_5GenerateModel<'a>),
     Qwen3ASR(Qwen3AsrGenerateModel<'a>),
-    Qwen3VL(Qwen3VLGenerateModel<'a>),
+    Qwen3VL(Box<Qwen3VLGenerateModel<'a>>),
     DeepSeekOCR(DeepseekOCRGenerateModel),
     HunyuanOCR(HunyuanOCRGenerateModel<'a>),
     PaddleOCRVL(Box<PaddleOCRVLGenerateModel<'a>>),
@@ -273,19 +273,19 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_
         }
         WhichModel::Qwen3vl2B => {
             let model = Qwen3VLGenerateModel::init(path, None, None)?;
-            ModelInstance::Qwen3VL(model)
+            ModelInstance::Qwen3VL(Box::new(model))
         }
         WhichModel::Qwen3vl4B => {
             let model = Qwen3VLGenerateModel::init(path, None, None)?;
-            ModelInstance::Qwen3VL(model)
+            ModelInstance::Qwen3VL(Box::new(model))
         }
         WhichModel::Qwen3vl8B => {
             let model = Qwen3VLGenerateModel::init(path, None, None)?;
-            ModelInstance::Qwen3VL(model)
+            ModelInstance::Qwen3VL(Box::new(model))
         }
         WhichModel::Qwen3vl32B => {
             let model = Qwen3VLGenerateModel::init(path, None, None)?;
-            ModelInstance::Qwen3VL(model)
+            ModelInstance::Qwen3VL(Box::new(model))
         }
         WhichModel::DeepSeekOCR => {
             let model = DeepseekOCRGenerateModel::init(path, None, None)?;

src/models/qwen3_5/generate.rs

@@ -25,7 +25,7 @@ use crate::{
 pub struct Qwen3_5GenerateModel<'a> {
     chat_template: ChatTemplate<'a>,
     tokenizer: TokenizerModel,
-    pre_processor: Qwen3VLProcessor,
+    pre_processor: Option<Qwen3VLProcessor>,
     qwen3_5: Qwen3_5Model,
     device: Device,
     eos_token_id: u32,
@@ -57,7 +57,7 @@ impl<'a> Qwen3_5GenerateModel<'a> {
         Ok(Self {
             chat_template,
             tokenizer,
-            pre_processor,
+            pre_processor: Some(pre_processor),
             qwen3_5,
             device,
             eos_token_id,
@@ -84,26 +84,26 @@ impl<'a> Qwen3_5GenerateModel<'a> {
         let mut reader = std::fs::File::open(model_file)?;
         let content = gguf_file::Content::read(&mut reader)?;
         let device = get_device(device);
-        let mut gguf = Gguf::new(content, reader, device.clone());
+        let mut model_gguf = Gguf::new(content, reader, device.clone());
 
-        let chat_template_str = gguf
+        let chat_template_str = model_gguf
             .get_matedata("tokenizer.chat_template")?
             .to_string()?
             .clone();
         let chat_template = ChatTemplate::str_init(&chat_template_str)?;
-        let tokenizer = gguf.build_tokenizer(Some(false), Some(false), Some(false))?;
+        let tokenizer = model_gguf.build_tokenizer(Some(false), Some(false), Some(false))?;
-        let dtype = match gguf.get_matedata("general.type") {
+        let (pre_processor, mut mmproj_gguf) = if let Some(mmproj_f) = mmproj_file {
-            Ok(v) => match v.to_u32() {
+            let mut reader = std::fs::File::open(mmproj_f)?;
-                Ok(0) => DType::F32,
+            let content = gguf_file::Content::read(&mut reader)?;
-                Ok(1) => DType::F16,
+            let mmproj_gguf = Gguf::new(content, reader, device.clone());
-                _ => DType::F16,
+            let processor = Qwen3VLProcessor::new_qwen3_5_default(&device, DType::F32)?;
-            },
+            (Some(processor), Some(mmproj_gguf))
-            Err(_) => DType::F16,
+        } else {
+            (None, None)
         };
-        let pre_processor = Qwen3VLProcessor::new_qwen3_5_default(&device, dtype)?;
 
         // let eos_token_id = gguf.get_matedata("tokenizer.ggml.eos_token_id")?.to_u32()?;
-        let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut gguf, &device)?;
+        let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut model_gguf, mmproj_gguf.as_mut(), &device)?;
         let stem = std::path::Path::new(model_file)
             .file_stem() // 获取文件名主干（不含扩展名）
             .and_then(|s| s.to_str())
@@ -131,18 +131,28 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
             get_logit_processor(temperature.into(), top_p.into(), Some(20), seed);
         // let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
         let mes_render = self.chat_template.apply_chat_template(&mes)?;
-
+        let (mes_text, pixel_values, image_grid_thw, pixel_values_video, video_grid_thw) =
-        let input = self.pre_processor.process_info(&mes, &mes_render)?;
+            if let Some(processor) = &self.pre_processor {
-        let mut input_ids = self
+                let input = processor.process_info(&mes, &mes_render)?;
-            .tokenizer
+                (
-            .text_encode(input.replace_text.clone(), &self.device)?;
+                    input.replace_text,
+                    input.pixel_values,
+                    input.image_grid_thw,
+                    input.pixel_values_video,
+                    input.video_grid_thw,
+                )
+            } else {
+                (mes_render, None, None, None, None)
+            };
+        // let input = self.pre_processor.process_info(&mes, &mes_render)?;
+        let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?;
         let mut seq_len = input_ids.dim(1)?;
         let prompt_tokens = seq_len as u32;
         let mut seqlen_offset = 0;
-        let mut pixel_values = input.pixel_values.as_ref();
+        let mut pixel_values = pixel_values.as_ref();
-        let image_grid_thw = input.image_grid_thw.as_ref();
+        let image_grid_thw = image_grid_thw.as_ref();
-        let mut pixel_values_video = input.pixel_values_video.as_ref();
+        let mut pixel_values_video = pixel_values_video.as_ref();
-        let video_grid_thw = input.video_grid_thw.as_ref();
+        let video_grid_thw = video_grid_thw.as_ref();
         let mut generate = Vec::new();
         let sample_len = mes.max_tokens.unwrap_or(1024);
         for _ in 0..sample_len {
@@ -202,19 +212,30 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
         let seed = mes.seed.unwrap_or(34562) as u64;
         let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
         let mes_render = self.chat_template.apply_chat_template(&mes)?;
-        let input = self.pre_processor.process_info(&mes, &mes_render)?;
+        // let input = self.pre_processor.process_info(&mes, &mes_render)?;
-        let mut input_ids = self
+        let (mes_text, pixel_values, image_grid_thw, pixel_values_video, video_grid_thw) =
-            .tokenizer
+            if let Some(processor) = &self.pre_processor {
-            .text_encode(input.replace_text.clone(), &self.device)?;
+                let input = processor.process_info(&mes, &mes_render)?;
+                (
+                    input.replace_text,
+                    input.pixel_values,
+                    input.image_grid_thw,
+                    input.pixel_values_video,
+                    input.video_grid_thw,
+                )
+            } else {
+                (mes_render, None, None, None, None)
+            };
+        let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?;
         let mut seq_len = input_ids.dim(1)?;
         let mut seqlen_offset = 0;
         let sample_len = mes.max_tokens.unwrap_or(1024);
         let stream = stream! {
             let mut error_tokens = Vec::new();
-            let mut pixel_values = input.pixel_values.as_ref();
+            let mut pixel_values = pixel_values.as_ref();
-            let image_grid_thw = input.image_grid_thw.as_ref();
+            let image_grid_thw = image_grid_thw.as_ref();
-            let mut pixel_values_video = input.pixel_values_video.as_ref();
+            let mut pixel_values_video = pixel_values_video.as_ref();
-            let video_grid_thw = input.video_grid_thw.as_ref();
+            let video_grid_thw = video_grid_thw.as_ref();
             let mut tool_call_id = None;
             let mut tool_call_content = String::new();
             let mut generate = Vec::new();

src/models/qwen3_5/model.rs

@@ -11,7 +11,7 @@ use crate::{
     models::{
         common::{
             conv1d_depthwise, eager_attention_forward, get_conv1d,
-            gguf::{GateUpDownMLPGguf, Gguf, ProjKind},
+            gguf::{GateUpDownMLPGguf, Gguf, ProjKind, QuantizedLinear},
             softplus,
         },
         qwen3_5::config::{Qwen3_5Config, Qwen3_5TextConfig},
@@ -55,24 +55,29 @@ impl Qwen3_5RMSNorm {
 
 pub struct Qwen3_5RMSNormGated {
     norm: RmsNorm,
+    dtype: DType,
 }
 
 impl Qwen3_5RMSNormGated {
     pub fn new(vb: VarBuilder, hidden_size: usize, eps: f64) -> Result<Self> {
+        let dtype = vb.dtype();
         let norm = rms_norm(hidden_size, eps, vb)?;
-        Ok(Self { norm })
+        Ok(Self { norm, dtype })
     }
 
     pub fn from_weight(weight: Tensor, eps: f64) -> Result<Self> {
+        let dtype = weight.dtype();
         let norm = RmsNorm::new(weight, eps);
-        Ok(Self { norm })
+        Ok(Self { norm, dtype })
     }
 
     pub fn forward(&self, xs: &Tensor, gate: Option<&Tensor>) -> Result<Tensor> {
-        let mut xs = self.norm.forward(xs)?;
+        let orig_dtype = xs.dtype();
+        let mut xs = self.norm.forward(&xs.to_dtype(self.dtype)?)?;
         if let Some(gate) = gate {
-            xs = xs.broadcast_mul(&gate.silu()?)?;
+            xs = xs.broadcast_mul(&gate.silu()?.to_dtype(xs.dtype())?)?;
         }
+        xs = xs.to_dtype(orig_dtype)?;
         Ok(xs)
     }
 }
@@ -225,11 +230,16 @@ impl Qwen3_5GatedDeltaNet {
         let a_log = gguf.get_dequantized(&format!("{prefix}.ssm_a"))?;
         let norm_weight = gguf.get_dequantized(&format!("{prefix}.ssm_norm.weight"))?;
         let norm = Qwen3_5RMSNormGated::from_weight(norm_weight, rms_norm_eps)?;
-        let out_proj = gguf.qmatmul(&format!("{prefix}.ssm_out.weight"))?;
+        // let out_proj = gguf.qmatmul(&format!("{prefix}.ssm_out.weight"))?;
-        let in_proj_qkv = gguf.qmatmul(&format!("{prefix}.attn_qkv.weight"))?;
+        // let in_proj_qkv = gguf.qmatmul(&format!("{prefix}.attn_qkv.weight"))?;
-        let in_proj_z = gguf.qmatmul(&format!("{prefix}.attn_gate.weight"))?;
+        // let in_proj_z = gguf.qmatmul(&format!("{prefix}.attn_gate.weight"))?;
-        let in_proj_b = gguf.qmatmul(&format!("{prefix}.ssm_beta.weight"))?;
+        // let in_proj_b = gguf.qmatmul(&format!("{prefix}.ssm_beta.weight"))?;
-        let in_proj_a = gguf.qmatmul(&format!("{prefix}.ssm_alpha.weight"))?;
+        // let in_proj_a = gguf.qmatmul(&format!("{prefix}.ssm_alpha.weight"))?;
+        let out_proj = gguf.quantize_linear(&format!("{prefix}.ssm_out"), false)?;
+        let in_proj_qkv = gguf.quantize_linear(&format!("{prefix}.attn_qkv"), false)?;
+        let in_proj_z = gguf.quantize_linear(&format!("{prefix}.attn_gate"), false)?;
+        let in_proj_b = gguf.quantize_linear(&format!("{prefix}.ssm_beta"), false)?;
+        let in_proj_a = gguf.quantize_linear(&format!("{prefix}.ssm_alpha"), false)?;
 
         Ok(Self {
             num_v_heads,
@@ -497,7 +507,7 @@ impl Qwen3_5GatedDeltaNet {
 
     pub fn forward(&mut self, xs: &Tensor, attention_mask: Option<&Tensor>) -> Result<Tensor> {
         let xs = if let Some(mask) = attention_mask {
-            xs.broadcast_mul(&mask.unsqueeze(D::Minus1)?)?
+            xs.broadcast_mul(&mask.unsqueeze(D::Minus1)?.to_dtype(xs.dtype())?)?
         } else {
             xs.clone()
         };
@@ -654,10 +664,14 @@ impl Qwen3_5Attention {
         let num_kv_groups = num_attention_heads / num_key_value_heads;
         let head_dim = gguf.get_matedata("qwen35.attention.key_length")?.to_u32()? as usize;
         let scaling = 1f64 / f64::sqrt(head_dim as f64);
-        let q_proj = gguf.qmatmul(&format!("{prefix}.attn_q.weight"))?;
+        // let q_proj = gguf.qmatmul(&format!("{prefix}.attn_q.weight"))?;
-        let k_proj = gguf.qmatmul(&format!("{prefix}.attn_k.weight"))?;
+        // let k_proj = gguf.qmatmul(&format!("{prefix}.attn_k.weight"))?;
-        let v_proj = gguf.qmatmul(&format!("{prefix}.attn_v.weight"))?;
+        // let v_proj = gguf.qmatmul(&format!("{prefix}.attn_v.weight"))?;
-        let o_proj = gguf.qmatmul(&format!("{prefix}.attn_output.weight"))?;
+        // let o_proj = gguf.qmatmul(&format!("{prefix}.attn_output.weight"))?;
+        let q_proj = gguf.quantize_linear(&format!("{prefix}.attn_q"), false)?;
+        let k_proj = gguf.quantize_linear(&format!("{prefix}.attn_k"), false)?;
+        let v_proj = gguf.quantize_linear(&format!("{prefix}.attn_v"), false)?;
+        let o_proj = gguf.quantize_linear(&format!("{prefix}.attn_output"), false)?;
         let q_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_q_norm.weight"))?;
         let q_norm = Qwen3_5RMSNorm::from_weight(q_norm_weight, rms_norm_eps)?;
         let k_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_k_norm.weight"))?;
@@ -800,6 +814,7 @@ impl Qwen3_5DecoderLayer {
             None,
             None,
             None,
+            Some(config.hidden_act),
         )?;
         let input_layernorm =
             Qwen3_5RMSNorm::new(vb.pp("input_layernorm"), hidden_size, config.rms_norm_eps)?;
@@ -831,7 +846,15 @@ impl Qwen3_5DecoderLayer {
             let attn = Qwen3_5Attention::new_from_gguf(gguf, prefix, rms_norm_eps)?;
             AttnKind::SelfAttn(attn)
         };
-        let mlp = GateUpDownMLPGguf::new_from_gguf(gguf, prefix)?;
+        let mlp = GateUpDownMLPGguf::new_from_gguf(
+            gguf,
+            prefix,
+            false,
+            None,
+            None,
+            None,
+            Some(candle_nn::Activation::Silu),
+        )?;
         let input_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_norm.weight"))?;
         let input_layernorm = Qwen3_5RMSNorm::from_weight(input_norm_weight, rms_norm_eps)?;
         let post_norm_weight =
@@ -857,16 +880,6 @@ impl Qwen3_5DecoderLayer {
         let residual = xs.clone();
         let mut xs = self.input_layernorm.forward(xs)?;
         xs = self.attn.forward(&xs, cos, sin, attention_mask)?;
-        // if self.layer_type.eq("linear_attention")
-        //     && let Some(linear_attn) = self.linear_attn.as_mut()
-        // {
-        //     xs = linear_attn.forward(&xs, attention_mask)?;
-        // } else if let Some(self_attn) = self.self_attn.as_mut()
-        //     && let Some(cos) = cos
-        //     && let Some(sin) = sin
-        // {
-        //     xs = self_attn.forward(&xs, cos, sin, attention_mask)?;
-        // }
         let residual = xs.add(&residual)?;
         xs = self.post_attention_layernorm.forward(&residual)?;
         xs = self.mlp.forward(&xs)?;
@@ -920,6 +933,14 @@ impl Qwen3_5TextModel {
         })
     }
     pub fn new_from_gguf<R: Read + Seek>(gguf: &mut Gguf<R>, device: &Device) -> Result<Self> {
+        let dtype = match gguf.get_matedata("general.dtype") {
+            Ok(v) => match v.to_u32() {
+                Ok(0) => DType::F32,
+                Ok(1) => DType::F16,
+                _ => DType::F16,
+            },
+            Err(_) => DType::F16,
+        };
         let num_layers = gguf.get_matedata("qwen35.block_count")?.to_u32()? as usize;
         let full_attention_interval = gguf
             .get_matedata("qwen35.full_attention_interval")?
@@ -939,6 +960,10 @@ impl Qwen3_5TextModel {
             .to_f32()? as f64;
         let hidden_size = gguf.get_matedata("qwen35.embedding_length")?.to_u32()? as usize; // 1024
         let embed_tensor = gguf.tensor("token_embd.weight")?;
+        // let embed_tokens = match dtype {
+        //     DType::F32 => Embedding::new(embed_tensor.dequantize(device)?, hidden_size),
+        //     _ => Embedding::new(embed_tensor.dequantize_f16(device)?, hidden_size),
+        // };
         let embed_tokens = Embedding::new(embed_tensor.dequantize(device)?, hidden_size);
         let mut layers = vec![];
         for i in 0..num_layers {
@@ -956,14 +981,7 @@ impl Qwen3_5TextModel {
         let norm_weight = gguf.get_dequantized("output_norm.weight")?;
         let norm = Qwen3_5RMSNorm::from_weight(norm_weight, rms_norm_eps)?;
         let rotary_emb = Qwen3VLTextRotaryEmbedding::new(rope_dimension_count, rope_freq_base);
-        let dtype = match gguf.get_matedata("general.type") {
+
-            Ok(v) => match v.to_u32() {
-                Ok(0) => DType::F32,
-                Ok(1) => DType::F16,
-                _ => DType::F16,
-            },
-            Err(_) => DType::F16,
-        };
         Ok(Self {
             embed_tokens,
             layers,
@@ -993,6 +1011,7 @@ impl Qwen3_5TextModel {
                 )?)
             }
         };
+        // let mut i = 0;
         for layer in self.layers.iter_mut() {
             let layer_mask =
                 if layer.layer_type.ne("linear_attention") || (seq_len != 1 && b_size != 1) {
@@ -1001,8 +1020,11 @@ impl Qwen3_5TextModel {
                     None
                 };
             xs = layer.forward(&xs, Some(&cos), Some(&sin), layer_mask.as_ref())?;
+            // println!("layer {i} : {}", xs);
+            // i += 1;
         }
         xs = self.norm.forward(&xs)?;
+        // println!("norm : {}", xs);
         Ok(xs)
     }
 
@@ -1052,11 +1074,21 @@ impl Qwen3_5Model {
         })
     }
 
-    pub fn new_from_gguf<R: Read + Seek>(gguf: &mut Gguf<R>, device: &Device) -> Result<Self> {
+    pub fn new_from_gguf<R: Read + Seek>(
+        gguf: &mut Gguf<R>,
+        mmproj_gguf: Option<&mut Gguf<R>>,
+        device: &Device,
+    ) -> Result<Self> {
         let spatial_merge_size = 2usize;
         let image_token_id = 248056u32;
         let video_token_id = 248057u32;
         let vision_start_token_id = 248053u32;
+        let visual = if let Some(mmproj) = mmproj_gguf {
+            let visual = Qwen3VLVisionModel::new_from_gguf(mmproj)?;
+            Some(visual)
+        } else {
+            None
+        };
         let language_model = Qwen3_5TextModel::new_from_gguf(gguf, device)?;
         let lm_head_tensor = match gguf.tensor("output.weight") {
             Ok(tensor) => tensor,
@@ -1068,9 +1100,9 @@ impl Qwen3_5Model {
             image_token_id,
             video_token_id,
             vision_start_token_id,
-            visual: None,
+            visual,
             language_model,
-            lm_head: ProjKind::QuantizedProj(lm_head),
+            lm_head: ProjKind::QuantizedProj(QuantizedLinear::new(lm_head, None)),
             rope_deltas: None,
         })
     }
@@ -1351,6 +1383,7 @@ impl Qwen3_5Model {
         seqlen_offset: usize,
     ) -> Result<Tensor> {
         let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?;
+        // println!("embed_tokens: {}", inputs_embeds);
         if let Some(pixel_values) = pixel_values
             && let Some(image_grid_thw) = image_grid_thw
             && let Some(visual) = self.visual.as_ref()
@@ -1365,6 +1398,7 @@ impl Qwen3_5Model {
                     image_embeds.dim(0)?
                 )));
             }
+            let image_embeds = image_embeds.to_dtype(inputs_embeds.dtype())?;
             inputs_embeds = masked_scatter_dim0(&inputs_embeds, &image_embeds, &vision_mask)?;
         }
         if let Some(pixel_values_video) = pixel_values_video
@@ -1381,9 +1415,10 @@ impl Qwen3_5Model {
                     video_embeds.dim(0)?
                 )));
             }
+            let video_embeds = video_embeds.to_dtype(inputs_embeds.dtype())?;
             inputs_embeds = masked_scatter_dim0(&inputs_embeds, &video_embeds, &vision_mask)?;
         }
-
+        // println!("visual : {}", inputs_embeds);
         let position_ids = self.compute_3d_position_ids(
             input_ids,
             &inputs_embeds,
@@ -1394,7 +1429,9 @@ impl Qwen3_5Model {
         let outputs = self.language_model.forward(&inputs_embeds, &position_ids)?;
         let seq_len = outputs.dim(1)?;
         let hidden_state = outputs.narrow(1, seq_len - 1, 1)?;
+        // println!("narrow 1 : {}", hidden_state);
         let logits = self.lm_head.forward(&hidden_state)?;
+        // println!("logits : {}", logits);
         Ok(logits)
     }
 

src/models/qwen3vl/model.rs

@@ -1,3 +1,5 @@
+use std::io::{Read, Seek};
+
 use anyhow::{Result, anyhow};
 use candle_core::{D, DType, IndexOp, Shape, Tensor};
 use candle_nn::{
@@ -7,7 +9,10 @@ use candle_nn::{
 
 use crate::{
     models::{
-        common::{TwoLinearMLP, eager_attention_forward, get_layer_norm},
+        common::{
+            eager_attention_forward, get_layer_norm,
+            gguf::{Gguf, ProjKind, TwoLinearMLPGguf},
+        },
         qwen3::model::Qwen3DecoderLayer,
         qwen3vl::config::{
             Qwen3VLConfig, Qwen3VLTextConfig, Qwen3VLVisionConfig, qwen3vl_text_config2qwen3_config,
@@ -60,6 +65,33 @@ impl Qwen3VLVisionPatchEmbed {
         })
     }
 
+    pub fn new_from_gguf<R: Read + Seek>(gguf: &mut Gguf<R>) -> Result<Self> {
+        // convert_hf_to_gguf.py
+        // temporal_patch_size = 2
+        //  spilt (embed_dim, in_channels, temporal_patch_size, patch_size, patch_size)
+        //  into two (embed_dim, in_channels, patch_size, patch_size)
+        // elif 'patch_embed.proj.weight' in name:
+        //         # split Conv3D into Conv2Ds
+        //         c1, c2, kt, kh, kw = data_torch.shape
+        //         del c1, c2, kh, kw  # unused
+        //         assert kt == 2, "Current implementation only support temporal_patch_size of 2"
+        //         yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight"  , data_torch[:, :, 0, ...])
+        //         yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...])
+        // (embed_dim, in_channels, patch_size, patch_size)  -> (embed_dim, in_channels, 1, patch_size, patch_size)
+        let conv3d_weight_0 = gguf.get_dequantized("v.patch_embd.weight")?.unsqueeze(2)?;
+        let conv3d_weight_1 = gguf
+            .get_dequantized("v.patch_embd.weight.1")?
+            .unsqueeze(2)?;
+        let conv3d_weight = Tensor::cat(&[conv3d_weight_0, conv3d_weight_1], 2)?
+            .flatten(1, 4)?
+            .t()?;
+        let conv3d_bias = gguf.get_dequantized("v.patch_embd.bias")?;
+        Ok(Self {
+            conv3d_weight,
+            conv3d_bias,
+        })
+    }
+
     pub fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
         // hidden_states shape:  (grid_t*grid_h*grid_w, c*temporal_patch_size*patch_size*patch_size)
         // ((), 1536) matmul (1536, 1024) -> ((), 1024)
@@ -73,9 +105,11 @@ pub struct Qwen3VLVisionPatchMerger {
     hidden_size: usize,
     use_postshuffle_norm: bool,
     norm: LayerNorm,
-    linear_fc1: Linear,
+    // linear_fc1: Linear,
+    linear_fc1: ProjKind,
     act_fn: Activation,
-    linear_fc2: Linear,
+    // linear_fc2: Linear,
+    linear_fc2: ProjKind,
 }
 
 impl Qwen3VLVisionPatchMerger {
@@ -98,9 +132,34 @@ impl Qwen3VLVisionPatchMerger {
             hidden_size,
             use_postshuffle_norm,
             norm,
-            linear_fc1,
+            linear_fc1: ProjKind::LinearProj(linear_fc1),
             act_fn,
-            linear_fc2,
+            linear_fc2: ProjKind::LinearProj(linear_fc2),
+        })
+    }
+
+    pub fn new_from_gguf<R: Read + Seek>(
+        gguf: &mut Gguf<R>,
+        rms_norm_eps: f64,
+        use_postshuffle_norm: bool,
+        hidden_size: usize,
+        spatial_merge_size: usize,
+        norm_prefix: &str,
+        linear1_prefix: &str,
+        linear2_prefix: &str,
+    ) -> Result<Self> {
+        let hidden_size = hidden_size * spatial_merge_size.pow(2);
+        let norm = gguf.layer_norm(norm_prefix, rms_norm_eps)?;
+        let linear_1 = gguf.quantize_linear(linear1_prefix, true)?;
+        let act_fn = Activation::Gelu;
+        let linear_2 = gguf.quantize_linear(linear2_prefix, true)?;
+        Ok(Self {
+            hidden_size,
+            use_postshuffle_norm,
+            norm,
+            linear_fc1: ProjKind::QuantizedProj(linear_1),
+            act_fn,
+            linear_fc2: ProjKind::QuantizedProj(linear_2),
         })
     }
 
@@ -120,8 +179,10 @@ impl Qwen3VLVisionPatchMerger {
 
 pub struct Qwen3VLVisionAttention {
     num_heads: usize,
-    qkv: Linear,
+    // qkv: Linear,
-    proj: Linear,
+    // proj: Linear,
+    qkv: ProjKind,
+    proj: ProjKind,
     scaling: f64,
 }
 
@@ -136,8 +197,27 @@ impl Qwen3VLVisionAttention {
 
         Ok(Self {
             num_heads,
-            qkv,
+            qkv: ProjKind::LinearProj(qkv),
-            proj,
+            proj: ProjKind::LinearProj(proj),
+            scaling,
+        })
+    }
+
+    pub fn new_from_gguf<R: Read + Seek>(mmproj_gguf: &mut Gguf<R>, prefix: &str) -> Result<Self> {
+        let num_heads = mmproj_gguf
+            .get_matedata("clip.vision.attention.head_count")?
+            .to_u32()? as usize;
+        let hidden_size = mmproj_gguf
+            .get_matedata("clip.vision.embedding_length")?
+            .to_u32()? as usize;
+        let head_dim = hidden_size / num_heads;
+        let scaling = 1.0 / (head_dim as f64).sqrt();
+        let qkv = mmproj_gguf.quantize_linear(&format!("{prefix}.attn_qkv"), true)?;
+        let proj = mmproj_gguf.quantize_linear(&format!("{prefix}.attn_out"), true)?;
+        Ok(Self {
+            num_heads,
+            qkv: ProjKind::QuantizedProj(qkv),
+            proj: ProjKind::QuantizedProj(proj),
             scaling,
         })
     }
@@ -195,7 +275,8 @@ pub struct Qwen3VLVisionBlock {
     norm1: LayerNorm,
     norm2: LayerNorm,
     attn: Qwen3VLVisionAttention,
-    mlp: TwoLinearMLP,
+    // mlp: TwoLinearMLP,
+    mlp: TwoLinearMLPGguf,
 }
 
 impl Qwen3VLVisionBlock {
@@ -203,7 +284,17 @@ impl Qwen3VLVisionBlock {
         let norm1 = get_layer_norm(vb.pp("norm1"), 1e-6, config.hidden_size, true)?;
         let norm2 = get_layer_norm(vb.pp("norm2"), 1e-6, config.hidden_size, true)?;
         let attn = Qwen3VLVisionAttention::new(config.clone(), vb.pp("attn"))?;
-        let mlp = TwoLinearMLP::new(
+        // let mlp = TwoLinearMLP::new(
+        //     vb.pp("mlp"),
+        //     config.hidden_size,
+        //     config.intermediate_size,
+        //     config.hidden_size,
+        //     config.hidden_act,
+        //     true,
+        //     "linear_fc1",
+        //     "linear_fc2",
+        // )?;
+        let mlp = TwoLinearMLPGguf::new(
             vb.pp("mlp"),
             config.hidden_size,
             config.intermediate_size,
@@ -221,6 +312,30 @@ impl Qwen3VLVisionBlock {
         })
     }
 
+    pub fn new_from_gguf<R: Read + Seek>(
+        mmproj_gguf: &mut Gguf<R>,
+        prefix: &str,
+        rms_norm_eps: f64,
+    ) -> Result<Self> {
+        let norm1 = mmproj_gguf.layer_norm(&format!("{prefix}.ln1"), rms_norm_eps)?;
+        let norm2 = mmproj_gguf.layer_norm(&format!("{prefix}.ln2"), rms_norm_eps)?;
+        let attn = Qwen3VLVisionAttention::new_from_gguf(mmproj_gguf, prefix)?;
+        let mlp = TwoLinearMLPGguf::new_from_gguf(
+            mmproj_gguf,
+            prefix,
+            true,
+            Some("ffn_up"),
+            Some("ffn_down"),
+            Some(Activation::GeluPytorchTanh),
+        )?;
+        Ok(Self {
+            norm1,
+            norm2,
+            attn,
+            mlp,
+        })
+    }
+
     pub fn forward(
         &self,
         xs: &Tensor,
@@ -292,6 +407,92 @@ impl Qwen3VLVisionModel {
         })
     }
 
+    pub fn new_from_gguf<R: Read + Seek>(mmproj_gguf: &mut Gguf<R>) -> Result<Self> {
+        // let num_layers = gguf.get_matedata("qwen35.block_count")?.to_u32()? as usize;
+        let spatial_merge_size = mmproj_gguf
+            .get_matedata("clip.vision.spatial_merge_size")?
+            .to_u32()? as usize;
+        let patch_embed = Qwen3VLVisionPatchEmbed::new_from_gguf(mmproj_gguf)?;
+        let pos_emb_weight = mmproj_gguf.get_dequantized("v.position_embd.weight")?;
+        let hidden_size = mmproj_gguf
+            .get_matedata("clip.vision.embedding_length")?
+            .to_u32()? as usize;
+        let pos_embed = Embedding::new(pos_emb_weight, hidden_size);
+        let patch_size = mmproj_gguf
+            .get_matedata("clip.vision.patch_size")?
+            .to_u32()? as usize;
+        let image_size = mmproj_gguf
+            .get_matedata("clip.vision.image_size")?
+            .to_u32()? as usize;
+        let num_grid_per_side = image_size / patch_size;
+        let num_heads = mmproj_gguf
+            .get_matedata("clip.vision.attention.head_count")?
+            .to_u32()? as usize;
+        let head_dim = hidden_size / num_heads;
+        let rotary_pos_emb = Qwen2_5VisionRotaryEmbedding::new(head_dim / 2, None);
+        let rms_norm_eps = mmproj_gguf
+            .get_matedata("clip.vision.attention.layer_norm_epsilon")?
+            .to_f32()? as f64;
+        let mut blocks = Vec::new();
+        let num_block = mmproj_gguf
+            .get_matedata("clip.vision.block_count")?
+            .to_u32()? as usize;
+        for i in 0..num_block {
+            let prefix = format!("v.blk.{i}");
+            // let block = Qwen3VLVisionBlock::new(config.clone(), vb_blocks.pp(i))?;
+            let block = Qwen3VLVisionBlock::new_from_gguf(mmproj_gguf, &prefix, rms_norm_eps)?;
+            blocks.push(block);
+        }
+        let merger = Qwen3VLVisionPatchMerger::new_from_gguf(
+            mmproj_gguf,
+            rms_norm_eps,
+            false,
+            hidden_size,
+            spatial_merge_size,
+            "v.post_ln",
+            "mm.0",
+            "mm.2",
+        )?;
+        let mut deepstack_merger_list = Vec::new();
+        let is_deepstack = mmproj_gguf
+            .get_matedata("clip.vision.is_deepstack_layers")?
+            .to_vec()?
+            .iter()
+            .map(|b| b.to_bool())
+            .collect::<Result<Vec<bool>, candle_core::Error>>()?;
+        let deepstack_visual_indexes = is_deepstack
+            .iter()
+            .enumerate()
+            .filter_map(|(i, &b)| if b { Some(i) } else { None })
+            .collect::<Vec<usize>>();
+        for i in &deepstack_visual_indexes {
+            let prefix = format!("v.deepstack.{i}");
+            let merger_i = Qwen3VLVisionPatchMerger::new_from_gguf(
+                mmproj_gguf,
+                rms_norm_eps,
+                true,
+                hidden_size,
+                spatial_merge_size,
+                &format!("{prefix}.norm"),
+                &format!("{prefix}.fc1"),
+                &format!("{prefix}.fc2"),
+            )?;
+            deepstack_merger_list.push(merger_i);
+        }
+        Ok(Self {
+            spatial_merge_size,
+            patch_embed,
+            pos_embed,
+            num_grid_per_side: num_grid_per_side as u32,
+            rotary_pos_emb,
+            blocks,
+            merger,
+            deepstack_visual_indexes,
+            deepstack_merger_list,
+            dtype: DType::F32,
+        })
+    }
+
     pub fn fast_pos_embed_interpolate(&self, grid_thw: &Tensor) -> Result<Tensor> {
         let mut idx_list = vec![vec![]; 4];
         let mut weight_list = vec![vec![]; 4];

src/models/qwen3vl/processor.rs

@@ -101,7 +101,6 @@ impl Qwen3VLProcessor {
             max_frames: 768,
         })
     }
-
     pub fn new_qwen3_5_default(device: &Device, dtype: DType) -> Result<Self> {
         let img_process_cfg = PreprocessorConfig::qwen3_5_img_default();
         let video_process_cfg = PreprocessorConfig::qwen3_5_video_default();

tests/test_gguf_qwen3_5.rs

@@ -5,24 +5,40 @@ use aha::{
     models::{GenerateModel, qwen3_5::generate::Qwen3_5GenerateModel},
 };
 use anyhow::Result;
-// use candle_core::{Device, quantized::gguf_file};
+// use candle_core::{DType, Device, quantized::gguf_file};
 #[test]
 fn gguf_test() -> Result<()> {
-    // cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture
+    // RUST_BACKTRACE=1 cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture
-    // let path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题
+    // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题
-    // let path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf";
+    // let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/mmproj-F16.gguf";
-    let path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf";
+    // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf";
-    // let mut file = std::fs::File::open(path)?;
+    let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf";
-    // let model = gguf_file::Content::read(&mut file)?;
+    let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/mmproj-F16.gguf";
-    // println!("group_count: {:?}", model.metadata.get("qwen35.ssm.group_count"));
+    // let mut model_file = std::fs::File::open(model_path)?;
-    // println!("time_step_rank: {:?}", model.metadata.get("qwen35.ssm.time_step_rank"));
+    // let model = gguf_file::Content::read(&mut model_file)?;
-    // println!("state_size: {:?}", model.metadata.get("qwen35.ssm.state_size"));
     // for (key, value) in model.metadata {
-    //     if key.contains("tokenizer") {
+    //     if key.contains("tokeni") {
     //         continue;
     //     }
     //     println!("{key}: {:#?}", value);
     // }
+    // let mut mmproj_file = std::fs::File::open(mmproj_path)?;
+    // let mmproj = gguf_file::Content::read(&mut mmproj_file)?;
+    // println!("model: {:#?}", mmproj.tensor_infos.keys());
+    // println!("group_count: {:?}", model.metadata.get("qwen35.ssm.group_count"));
+    // println!("time_step_rank: {:?}", model.metadata.get("qwen35.ssm.time_step_rank"));
+    // println!("state_size: {:?}", model.metadata.get("qwen35.ssm.state_size"));
+    // for (key, value) in mmproj.metadata {
+    //     println!("{key}: {:#?}", value);
+    // }
+    // let device = Device::new_cuda(0)?;
+    // let mut mmproj_gguf = Gguf::new(mmproj, mmproj_file, device.clone());
+    // let weight = mmproj_gguf.get_dequantized("v.position_embd.weight")?;
+    // println!("weight: {:?}", weight);
+    // let conv3d_weight_1 = mmproj_gguf.get_dequantized("v.patch_embd.weight.1")?;
+    // println!("conv3d_weight_1: {}", conv3d_weight_1);
+    // let conv3d_bias = mmproj_gguf.get_dequantized("v.patch_embd.bias")?;
+    // println!("conv3d_bias: {}", conv3d_bias);
     // println!("model: {:?}", model.magic);
     // println!("generat.type: {:#?}", model.metadata.keys());
     // println!("tokenizer.ggml.eos_token_id: {:#?}", model.metadata.get("tokenizer.ggml.eos_token_id"));
@@ -33,19 +49,28 @@ fn gguf_test() -> Result<()> {
         "messages": [
             {
                 "role": "user",
-                "content": [        
+                "content": [
                     {
-                        "type": "text", 
+                        "type": "image",
-                        "text": "你如何看待AI"
+                        "image_url":
+                        {
+                            "url": "file://./assets/img/ocr_test1.png"
+                        }
+                    },
+                    {
+                        "type": "text",
+                        "text": "请分析图片并提取所有可见文本内容，按从左到右、从上到下的布局，返回纯文本"
                     }
                 ]
             }
         ]
     }
     "#;
+
     let mes: ChatCompletionParameters = serde_json::from_str(message)?;
     let i_start = Instant::now();
-    let mut gguf_qwen3_5 = Qwen3_5GenerateModel::init_from_gguf(path, None, None)?;
+    let mut gguf_qwen3_5 =
+        Qwen3_5GenerateModel::init_from_gguf(model_path, mmproj_path.into(), None)?;
     let i_duration = i_start.elapsed();
     println!("Time elapsed in load model is: {:?}", i_duration);
 

tests/test_qwen3_5.rs

@@ -24,12 +24,12 @@ fn qwen3_5_generate() -> Result<()> {
                         "type": "image",
                         "image_url": 
                         {
-                            "url": "file:///home/jhq/Downloads/gougou1.jpg"
+                            "url": "file://./assets/img/ocr_test1.png"
                         }
                     },             
                     {
                         "type": "text", 
-                        "text": "描述这张图片."
+                        "text": "OCR"
                     }
                 ]
             }

tests/test_qwen3vl.rs

@@ -29,7 +29,7 @@ fn qwen3vl_thinking_generate() -> Result<()> {
                     },             
                     {
                         "type": "text", 
-                        "text": "请分析图片并提取所有可见文本内容，按从左到右、从上到下的布局，返回纯文本"
+                        "text": "OCR"
                     }
                 ]
             }
@@ -59,7 +59,7 @@ fn qwen3vl_thinking_generate() -> Result<()> {
 
 #[test]
 fn qwen3vl_generate() -> Result<()> {
-    // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda,ffmpeg --test test_qwen3vl qwen3vl_generate -r -- --nocapture
+    // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_qwen3vl qwen3vl_generate -r -- --nocapture
 
     let save_dir =
         aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
@@ -73,15 +73,15 @@ fn qwen3vl_generate() -> Result<()> {
                 "role": "user",
                 "content": [
                     {
-                        "type": "video",
+                        "type": "image",
-                        "video_url": 
+                        "image_url": 
                         {
-                            "url": "./assets/video/video_test.mp4"
+                            "url": "file://./assets/img/ocr_test1.png"
                         }
-                    },              
+                    },             
                     {
                         "type": "text", 
-                        "text": "视频中发生了什么？"
+                        "text": "OCR"
                     }
                 ]
             }