From 80d36b5308dc2939d18066360d28c46851cd1c1d Mon Sep 17 00:00:00 2001 From: jhqxxx <18280426169@163.com> Date: Mon, 16 Mar 2026 14:50:59 +0800 Subject: [PATCH] add Qwen3.5 mmproj gguf --- README.md | 3 + README.zh-CN.md | 3 + docs/changelog.md | 3 + docs/changelog.zh-CN.md | 3 + src/models/common/gguf.rs | 183 +++++++++++++++++++++++--- src/models/common/mod.rs | 4 +- src/models/mod.rs | 10 +- src/models/qwen3_5/generate.rs | 83 +++++++----- src/models/qwen3_5/model.rs | 113 ++++++++++------ src/models/qwen3vl/model.rs | 223 ++++++++++++++++++++++++++++++-- src/models/qwen3vl/processor.rs | 1 - tests/test_gguf_qwen3_5.rs | 55 +++++--- tests/test_qwen3_5.rs | 4 +- tests/test_qwen3vl.rs | 14 +- 14 files changed, 575 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 39845f8..b91c663 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ aha is a high-performance, cross-platform AI inference engine built with Rust and the Candle framework. It brings state-of-the-art AI models to your local machine—no API keys, no cloud dependencies, just pure, fast AI running directly on your hardware. ## Changelog +### 2026-03-16 +- Added Qwen3.5 mmproj + ### 2026-03-14 - update rust version - Added Qwen3.5 gguf support, but the 4B model still has issues; to be resolved. diff --git a/README.zh-CN.md b/README.zh-CN.md index 43863d9..771f89d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -25,6 +25,9 @@ aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理引擎。将最先进的 AI 模型带到您的本地机器——无需 API 密钥，无需云依赖，纯粹、快速的 AI，直接在您的硬件上运行。 ## 更新日志 +### 2026-03-16 +- 增加 Qwen3.5 mmproj + ### 2026-03-14 - 更新rust版本 - 增加了对 Qwen3.5 gguf 的支持，但 4B 模型仍然存在问题；待解决。 diff --git a/docs/changelog.md b/docs/changelog.md index 0dffc8c..5f9f760 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,9 @@ All notable changes to aha will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +### 2026-03-16 +- Added Qwen3.5 mmproj + ### 2026-03-14 - update rust version - Added Qwen3.5 gguf support, but the 4B model still has issues; to be resolved. diff --git a/docs/changelog.zh-CN.md b/docs/changelog.zh-CN.md index 9f378d8..3fbff27 100644 --- a/docs/changelog.zh-CN.md +++ b/docs/changelog.zh-CN.md @@ -5,6 +5,9 @@ 格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)， 本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/spec/v2.0.0.html)。 +### 2026-03-16 +- 增加 Qwen3.5 mmproj + ### 2026-03-14 - 更新rust版本 - 增加了对 Qwen3.5 gguf 的支持，但 4B 模型仍然存在问题；待解决。 diff --git a/src/models/common/gguf.rs b/src/models/common/gguf.rs index cb54c27..684734d 100644 --- a/src/models/common/gguf.rs +++ b/src/models/common/gguf.rs @@ -3,13 +3,15 @@ use std::io::{Read, Seek}; use ahash::AHashMap; use anyhow::{Result, anyhow}; use candle_core::{ - Device, Tensor, + DType, Device, Tensor, quantized::{ QMatMul, QTensor, gguf_file::{self, Value}, }, }; -use candle_nn::{Conv1d, Conv1dConfig, Linear, Module, RmsNorm, VarBuilder, linear_b}; +use candle_nn::{ + Activation, Conv1d, Conv1dConfig, LayerNorm, Linear, Module, RmsNorm, VarBuilder, linear_b, +}; use tokenizers::{self, AddedToken, Tokenizer, models::bpe::BPE}; use crate::tokenizer::TokenizerModel; @@ -37,12 +39,40 @@ impl Gguf { Ok(QMatMul::from_qtensor(ws)?) } + pub fn quantize_linear(&mut self, prefix: &str, bias: bool) -> Result { + let weight = self.qmatmul(&format!("{prefix}.weight"))?; + let bias = if bias { + self.get_dequantized(&format!("{prefix}.bias")).ok() + } else { + None + }; + Ok(QuantizedLinear::new(weight, bias)) + } + pub fn rms_norm(&mut self, name: &str, eps: f64) -> Result { let ws = self.ct.tensor(&mut self.reader, name, &self.device)?; let weight = ws.dequantize(&self.device)?; Ok(RmsNorm::new(weight, eps)) } + pub fn layer_norm(&mut self, prefix: &str, eps: f64) -> Result { + let weight = self + .ct + .tensor(&mut self.reader, &format!("{prefix}.weight"), &self.device)?; + let weight = weight.dequantize(&self.device)?; + let bias = self + .ct + .tensor(&mut self.reader, &format!("{prefix}.bias"), &self.device); + let bias = match bias { + Ok(bias) => bias.dequantize(&self.device).ok(), + Err(_) => None, + }; + match bias { + Some(bias) => Ok(LayerNorm::new(weight, bias, eps)), + None => Ok(LayerNorm::new_no_bias(weight, eps)), + } + } + pub fn metadata(&self) -> &std::collections::HashMap { &self.ct.metadata } @@ -55,6 +85,10 @@ impl Gguf { Ok(self.tensor(name)?.dequantize(&self.device)?) } + pub fn get_dequantized_f16(&mut self, name: &str) -> Result { + Ok(self.tensor(name)?.dequantize_f16(&self.device)?) + } + pub fn conv1d( &mut self, prefix: &str, @@ -169,9 +203,36 @@ impl Gguf { } } +#[derive(Debug, Clone)] +pub struct QuantizedLinear { + inner: QMatMul, + bias: Option, +} + +impl QuantizedLinear { + pub fn new(inner: QMatMul, bias: Option) -> Self { + Self { inner, bias } + } +} + +impl Module for QuantizedLinear { + fn forward(&self, xs: &Tensor) -> candle_core::Result { + let xs = if xs.dtype() == DType::F16 { + self.inner.forward_via_f16(xs)? + } else { + self.inner.forward(xs)? + }; + if let Some(bias) = &self.bias { + xs.broadcast_add(&bias.to_dtype(xs.dtype())?) + } else { + Ok(xs) + } + } +} + #[derive(Debug, Clone)] pub enum ProjKind { - QuantizedProj(QMatMul), + QuantizedProj(QuantizedLinear), LinearProj(Linear), } @@ -186,20 +247,52 @@ impl Module for ProjKind { #[derive(Debug, Clone)] pub struct GateUpDownMLPGguf { - gate_proj: ProjKind, // ffn_gate.weight - up_proj: ProjKind, // ffn_up.weight - down_proj: ProjKind, // ffn_down.weight + gate_proj: ProjKind, + up_proj: ProjKind, + down_proj: ProjKind, + act: Activation, } impl GateUpDownMLPGguf { - pub fn new_from_gguf(gguf: &mut Gguf, prefix: &str) -> Result { - let gate_proj = gguf.qmatmul(&format!("{prefix}.ffn_gate.weight"))?; - let up_proj = gguf.qmatmul(&format!("{prefix}.ffn_up.weight"))?; - let down_proj = gguf.qmatmul(&format!("{prefix}.ffn_down.weight"))?; + pub fn new_from_gguf( + gguf: &mut Gguf, + prefix: &str, + bias: bool, + gate_name: Option<&str>, + up_name: Option<&str>, + down_name: Option<&str>, + act: Option, + ) -> Result { + let gate_name = gate_name.unwrap_or("ffn_gate"); + let up_name = up_name.unwrap_or("ffn_up"); + let down_name = down_name.unwrap_or("ffn_down"); + let gate_proj = gguf.qmatmul(&format!("{prefix}.{gate_name}.weight"))?; + let gate_bias = if bias { + gguf.get_dequantized(&format!("{prefix}.{gate_name}.bias")) + .ok() + } else { + None + }; + let up_proj = gguf.qmatmul(&format!("{prefix}.{up_name}.weight"))?; + let up_bias = if bias { + gguf.get_dequantized(&format!("{prefix}.{up_name}.bias")) + .ok() + } else { + None + }; + let down_proj = gguf.qmatmul(&format!("{prefix}.{down_name}.weight"))?; + let down_bias = if bias { + gguf.get_dequantized(&format!("{prefix}.{down_name}.bias")) + .ok() + } else { + None + }; + let act = act.unwrap_or(Activation::Silu); Ok(Self { - gate_proj: ProjKind::QuantizedProj(gate_proj), - up_proj: ProjKind::QuantizedProj(up_proj), - down_proj: ProjKind::QuantizedProj(down_proj), + gate_proj: ProjKind::QuantizedProj(QuantizedLinear::new(gate_proj, gate_bias)), + up_proj: ProjKind::QuantizedProj(QuantizedLinear::new(up_proj, up_bias)), + down_proj: ProjKind::QuantizedProj(QuantizedLinear::new(down_proj, down_bias)), + act, }) } pub fn new_from_vb( @@ -210,6 +303,7 @@ impl GateUpDownMLPGguf { gate_pp_name: Option<&str>, up_pp_name: Option<&str>, down_pp_name: Option<&str>, + act: Option, ) -> Result { let gate_pp_name = gate_pp_name.unwrap_or("gate_proj"); let up_pp_name = up_pp_name.unwrap_or("up_proj"); @@ -217,18 +311,77 @@ impl GateUpDownMLPGguf { let gate_proj = linear_b(hidden_size, intermediate_size, bias, vb.pp(gate_pp_name))?; let up_proj = linear_b(hidden_size, intermediate_size, bias, vb.pp(up_pp_name))?; let down_proj = linear_b(intermediate_size, hidden_size, bias, vb.pp(down_pp_name))?; + let act = act.unwrap_or(Activation::Silu); Ok(Self { gate_proj: ProjKind::LinearProj(gate_proj), up_proj: ProjKind::LinearProj(up_proj), down_proj: ProjKind::LinearProj(down_proj), + act, }) } } impl Module for GateUpDownMLPGguf { fn forward(&self, xs: &Tensor) -> candle_core::Result { - let w1 = self.gate_proj.forward(xs)?; + let w1 = self.gate_proj.forward(xs)?.apply(&self.act)?; let w3 = self.up_proj.forward(xs)?; - self.down_proj.forward(&(candle_nn::ops::silu(&w1)? * w3)?) + self.down_proj.forward(&(w1 * w3)?) + } +} + +pub struct TwoLinearMLPGguf { + linear1: ProjKind, + linear2: ProjKind, + act: Activation, +} + +impl TwoLinearMLPGguf { + pub fn new( + vb: VarBuilder, + // embedding_dim: usize, + // mlp_dim: usize, + in_dim: usize, + middle_dim: usize, + out_dim: usize, + act: Activation, + bias: bool, + linear1_pp_name: &str, + linear2_pp_name: &str, + ) -> Result { + let linear1 = linear_b(in_dim, middle_dim, bias, vb.pp(linear1_pp_name))?; + let linear2 = linear_b(middle_dim, out_dim, bias, vb.pp(linear2_pp_name))?; + + Ok(Self { + linear1: ProjKind::LinearProj(linear1), + linear2: ProjKind::LinearProj(linear2), + act, + }) + } + + pub fn new_from_gguf( + gguf: &mut Gguf, + prefix: &str, + bias: bool, + linear1_name: Option<&str>, + linear2_name: Option<&str>, + act: Option, + ) -> Result { + let linear1_name = linear1_name.unwrap_or("ffn_up"); + let linear2_name = linear2_name.unwrap_or("ffn_down"); + let linear1 = gguf.quantize_linear(&format!("{prefix}.{linear1_name}"), bias)?; + let linear2 = gguf.quantize_linear(&format!("{prefix}.{linear2_name}"), bias)?; + let act = act.unwrap_or(Activation::Silu); + Ok(Self { + linear1: ProjKind::QuantizedProj(linear1), + linear2: ProjKind::QuantizedProj(linear2), + act, + }) + } + pub fn forward(&self, xs: &Tensor) -> Result { + let xs = xs + .apply(&self.linear1)? + .apply(&self.act)? + .apply(&self.linear2)?; + Ok(xs) } } diff --git a/src/models/common/mod.rs b/src/models/common/mod.rs index 0a26948..8d871c2 100644 --- a/src/models/common/mod.rs +++ b/src/models/common/mod.rs @@ -1172,7 +1172,7 @@ pub fn conv1d_depthwise(input: &Tensor, weight: &Tensor, bias: Option<&Tensor>) // weight: (dim, 1, k) -> (dim, k) // input already padding let len_in = input.dim(2)?; - let weight = weight.squeeze(1)?; + let weight = weight.squeeze(1)?.to_dtype(input.dtype())?; let kernel_size = weight.dim(1)?; // len_out = (len_in - k + 2p) / s + 1, p = 0, s = 1 let len_out = len_in - kernel_size + 1; @@ -1189,7 +1189,7 @@ pub fn conv1d_depthwise(input: &Tensor, weight: &Tensor, bias: Option<&Tensor>) None => Ok(out), Some(bias) => { let b = bias.dims1()?; - let bias = bias.reshape((1, b, 1))?; + let bias = bias.reshape((1, b, 1))?.to_dtype(input.dtype())?; Ok(out.broadcast_add(&bias)?) } } diff --git a/src/models/mod.rs b/src/models/mod.rs index 5ea49f4..10c1f98 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -168,7 +168,7 @@ pub enum ModelInstance<'a> { Qwen3(Qwen3GenerateModel<'a>), Qwen3_5(Qwen3_5GenerateModel<'a>), Qwen3ASR(Qwen3AsrGenerateModel<'a>), - Qwen3VL(Qwen3VLGenerateModel<'a>), + Qwen3VL(Box>), DeepSeekOCR(DeepseekOCRGenerateModel), HunyuanOCR(HunyuanOCRGenerateModel<'a>), PaddleOCRVL(Box>), @@ -273,19 +273,19 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result { let model = Qwen3VLGenerateModel::init(path, None, None)?; - ModelInstance::Qwen3VL(model) + ModelInstance::Qwen3VL(Box::new(model)) } WhichModel::Qwen3vl4B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; - ModelInstance::Qwen3VL(model) + ModelInstance::Qwen3VL(Box::new(model)) } WhichModel::Qwen3vl8B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; - ModelInstance::Qwen3VL(model) + ModelInstance::Qwen3VL(Box::new(model)) } WhichModel::Qwen3vl32B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; - ModelInstance::Qwen3VL(model) + ModelInstance::Qwen3VL(Box::new(model)) } WhichModel::DeepSeekOCR => { let model = DeepseekOCRGenerateModel::init(path, None, None)?; diff --git a/src/models/qwen3_5/generate.rs b/src/models/qwen3_5/generate.rs index 15483df..ca52310 100644 --- a/src/models/qwen3_5/generate.rs +++ b/src/models/qwen3_5/generate.rs @@ -25,7 +25,7 @@ use crate::{ pub struct Qwen3_5GenerateModel<'a> { chat_template: ChatTemplate<'a>, tokenizer: TokenizerModel, - pre_processor: Qwen3VLProcessor, + pre_processor: Option, qwen3_5: Qwen3_5Model, device: Device, eos_token_id: u32, @@ -57,7 +57,7 @@ impl<'a> Qwen3_5GenerateModel<'a> { Ok(Self { chat_template, tokenizer, - pre_processor, + pre_processor: Some(pre_processor), qwen3_5, device, eos_token_id, @@ -84,26 +84,26 @@ impl<'a> Qwen3_5GenerateModel<'a> { let mut reader = std::fs::File::open(model_file)?; let content = gguf_file::Content::read(&mut reader)?; let device = get_device(device); - let mut gguf = Gguf::new(content, reader, device.clone()); + let mut model_gguf = Gguf::new(content, reader, device.clone()); - let chat_template_str = gguf + let chat_template_str = model_gguf .get_matedata("tokenizer.chat_template")? .to_string()? .clone(); let chat_template = ChatTemplate::str_init(&chat_template_str)?; - let tokenizer = gguf.build_tokenizer(Some(false), Some(false), Some(false))?; - let dtype = match gguf.get_matedata("general.type") { - Ok(v) => match v.to_u32() { - Ok(0) => DType::F32, - Ok(1) => DType::F16, - _ => DType::F16, - }, - Err(_) => DType::F16, + let tokenizer = model_gguf.build_tokenizer(Some(false), Some(false), Some(false))?; + let (pre_processor, mut mmproj_gguf) = if let Some(mmproj_f) = mmproj_file { + let mut reader = std::fs::File::open(mmproj_f)?; + let content = gguf_file::Content::read(&mut reader)?; + let mmproj_gguf = Gguf::new(content, reader, device.clone()); + let processor = Qwen3VLProcessor::new_qwen3_5_default(&device, DType::F32)?; + (Some(processor), Some(mmproj_gguf)) + } else { + (None, None) }; - let pre_processor = Qwen3VLProcessor::new_qwen3_5_default(&device, dtype)?; // let eos_token_id = gguf.get_matedata("tokenizer.ggml.eos_token_id")?.to_u32()?; - let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut gguf, &device)?; + let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut model_gguf, mmproj_gguf.as_mut(), &device)?; let stem = std::path::Path::new(model_file) .file_stem() // 获取文件名主干（不含扩展名） .and_then(|s| s.to_str()) @@ -131,18 +131,28 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> { get_logit_processor(temperature.into(), top_p.into(), Some(20), seed); // let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed); let mes_render = self.chat_template.apply_chat_template(&mes)?; - - let input = self.pre_processor.process_info(&mes, &mes_render)?; - let mut input_ids = self - .tokenizer - .text_encode(input.replace_text.clone(), &self.device)?; + let (mes_text, pixel_values, image_grid_thw, pixel_values_video, video_grid_thw) = + if let Some(processor) = &self.pre_processor { + let input = processor.process_info(&mes, &mes_render)?; + ( + input.replace_text, + input.pixel_values, + input.image_grid_thw, + input.pixel_values_video, + input.video_grid_thw, + ) + } else { + (mes_render, None, None, None, None) + }; + // let input = self.pre_processor.process_info(&mes, &mes_render)?; + let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?; let mut seq_len = input_ids.dim(1)?; let prompt_tokens = seq_len as u32; let mut seqlen_offset = 0; - let mut pixel_values = input.pixel_values.as_ref(); - let image_grid_thw = input.image_grid_thw.as_ref(); - let mut pixel_values_video = input.pixel_values_video.as_ref(); - let video_grid_thw = input.video_grid_thw.as_ref(); + let mut pixel_values = pixel_values.as_ref(); + let image_grid_thw = image_grid_thw.as_ref(); + let mut pixel_values_video = pixel_values_video.as_ref(); + let video_grid_thw = video_grid_thw.as_ref(); let mut generate = Vec::new(); let sample_len = mes.max_tokens.unwrap_or(1024); for _ in 0..sample_len { @@ -202,19 +212,30 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> { let seed = mes.seed.unwrap_or(34562) as u64; let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed); let mes_render = self.chat_template.apply_chat_template(&mes)?; - let input = self.pre_processor.process_info(&mes, &mes_render)?; - let mut input_ids = self - .tokenizer - .text_encode(input.replace_text.clone(), &self.device)?; + // let input = self.pre_processor.process_info(&mes, &mes_render)?; + let (mes_text, pixel_values, image_grid_thw, pixel_values_video, video_grid_thw) = + if let Some(processor) = &self.pre_processor { + let input = processor.process_info(&mes, &mes_render)?; + ( + input.replace_text, + input.pixel_values, + input.image_grid_thw, + input.pixel_values_video, + input.video_grid_thw, + ) + } else { + (mes_render, None, None, None, None) + }; + let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?; let mut seq_len = input_ids.dim(1)?; let mut seqlen_offset = 0; let sample_len = mes.max_tokens.unwrap_or(1024); let stream = stream! { let mut error_tokens = Vec::new(); - let mut pixel_values = input.pixel_values.as_ref(); - let image_grid_thw = input.image_grid_thw.as_ref(); - let mut pixel_values_video = input.pixel_values_video.as_ref(); - let video_grid_thw = input.video_grid_thw.as_ref(); + let mut pixel_values = pixel_values.as_ref(); + let image_grid_thw = image_grid_thw.as_ref(); + let mut pixel_values_video = pixel_values_video.as_ref(); + let video_grid_thw = video_grid_thw.as_ref(); let mut tool_call_id = None; let mut tool_call_content = String::new(); let mut generate = Vec::new(); diff --git a/src/models/qwen3_5/model.rs b/src/models/qwen3_5/model.rs index d3b9d82..7a254d9 100644 --- a/src/models/qwen3_5/model.rs +++ b/src/models/qwen3_5/model.rs @@ -11,7 +11,7 @@ use crate::{ models::{ common::{ conv1d_depthwise, eager_attention_forward, get_conv1d, - gguf::{GateUpDownMLPGguf, Gguf, ProjKind}, + gguf::{GateUpDownMLPGguf, Gguf, ProjKind, QuantizedLinear}, softplus, }, qwen3_5::config::{Qwen3_5Config, Qwen3_5TextConfig}, @@ -55,24 +55,29 @@ impl Qwen3_5RMSNorm { pub struct Qwen3_5RMSNormGated { norm: RmsNorm, + dtype: DType, } impl Qwen3_5RMSNormGated { pub fn new(vb: VarBuilder, hidden_size: usize, eps: f64) -> Result { + let dtype = vb.dtype(); let norm = rms_norm(hidden_size, eps, vb)?; - Ok(Self { norm }) + Ok(Self { norm, dtype }) } pub fn from_weight(weight: Tensor, eps: f64) -> Result { + let dtype = weight.dtype(); let norm = RmsNorm::new(weight, eps); - Ok(Self { norm }) + Ok(Self { norm, dtype }) } pub fn forward(&self, xs: &Tensor, gate: Option<&Tensor>) -> Result { - let mut xs = self.norm.forward(xs)?; + let orig_dtype = xs.dtype(); + let mut xs = self.norm.forward(&xs.to_dtype(self.dtype)?)?; if let Some(gate) = gate { - xs = xs.broadcast_mul(&gate.silu()?)?; + xs = xs.broadcast_mul(&gate.silu()?.to_dtype(xs.dtype())?)?; } + xs = xs.to_dtype(orig_dtype)?; Ok(xs) } } @@ -225,11 +230,16 @@ impl Qwen3_5GatedDeltaNet { let a_log = gguf.get_dequantized(&format!("{prefix}.ssm_a"))?; let norm_weight = gguf.get_dequantized(&format!("{prefix}.ssm_norm.weight"))?; let norm = Qwen3_5RMSNormGated::from_weight(norm_weight, rms_norm_eps)?; - let out_proj = gguf.qmatmul(&format!("{prefix}.ssm_out.weight"))?; - let in_proj_qkv = gguf.qmatmul(&format!("{prefix}.attn_qkv.weight"))?; - let in_proj_z = gguf.qmatmul(&format!("{prefix}.attn_gate.weight"))?; - let in_proj_b = gguf.qmatmul(&format!("{prefix}.ssm_beta.weight"))?; - let in_proj_a = gguf.qmatmul(&format!("{prefix}.ssm_alpha.weight"))?; + // let out_proj = gguf.qmatmul(&format!("{prefix}.ssm_out.weight"))?; + // let in_proj_qkv = gguf.qmatmul(&format!("{prefix}.attn_qkv.weight"))?; + // let in_proj_z = gguf.qmatmul(&format!("{prefix}.attn_gate.weight"))?; + // let in_proj_b = gguf.qmatmul(&format!("{prefix}.ssm_beta.weight"))?; + // let in_proj_a = gguf.qmatmul(&format!("{prefix}.ssm_alpha.weight"))?; + let out_proj = gguf.quantize_linear(&format!("{prefix}.ssm_out"), false)?; + let in_proj_qkv = gguf.quantize_linear(&format!("{prefix}.attn_qkv"), false)?; + let in_proj_z = gguf.quantize_linear(&format!("{prefix}.attn_gate"), false)?; + let in_proj_b = gguf.quantize_linear(&format!("{prefix}.ssm_beta"), false)?; + let in_proj_a = gguf.quantize_linear(&format!("{prefix}.ssm_alpha"), false)?; Ok(Self { num_v_heads, @@ -497,7 +507,7 @@ impl Qwen3_5GatedDeltaNet { pub fn forward(&mut self, xs: &Tensor, attention_mask: Option<&Tensor>) -> Result { let xs = if let Some(mask) = attention_mask { - xs.broadcast_mul(&mask.unsqueeze(D::Minus1)?)? + xs.broadcast_mul(&mask.unsqueeze(D::Minus1)?.to_dtype(xs.dtype())?)? } else { xs.clone() }; @@ -654,10 +664,14 @@ impl Qwen3_5Attention { let num_kv_groups = num_attention_heads / num_key_value_heads; let head_dim = gguf.get_matedata("qwen35.attention.key_length")?.to_u32()? as usize; let scaling = 1f64 / f64::sqrt(head_dim as f64); - let q_proj = gguf.qmatmul(&format!("{prefix}.attn_q.weight"))?; - let k_proj = gguf.qmatmul(&format!("{prefix}.attn_k.weight"))?; - let v_proj = gguf.qmatmul(&format!("{prefix}.attn_v.weight"))?; - let o_proj = gguf.qmatmul(&format!("{prefix}.attn_output.weight"))?; + // let q_proj = gguf.qmatmul(&format!("{prefix}.attn_q.weight"))?; + // let k_proj = gguf.qmatmul(&format!("{prefix}.attn_k.weight"))?; + // let v_proj = gguf.qmatmul(&format!("{prefix}.attn_v.weight"))?; + // let o_proj = gguf.qmatmul(&format!("{prefix}.attn_output.weight"))?; + let q_proj = gguf.quantize_linear(&format!("{prefix}.attn_q"), false)?; + let k_proj = gguf.quantize_linear(&format!("{prefix}.attn_k"), false)?; + let v_proj = gguf.quantize_linear(&format!("{prefix}.attn_v"), false)?; + let o_proj = gguf.quantize_linear(&format!("{prefix}.attn_output"), false)?; let q_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_q_norm.weight"))?; let q_norm = Qwen3_5RMSNorm::from_weight(q_norm_weight, rms_norm_eps)?; let k_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_k_norm.weight"))?; @@ -800,6 +814,7 @@ impl Qwen3_5DecoderLayer { None, None, None, + Some(config.hidden_act), )?; let input_layernorm = Qwen3_5RMSNorm::new(vb.pp("input_layernorm"), hidden_size, config.rms_norm_eps)?; @@ -831,7 +846,15 @@ impl Qwen3_5DecoderLayer { let attn = Qwen3_5Attention::new_from_gguf(gguf, prefix, rms_norm_eps)?; AttnKind::SelfAttn(attn) }; - let mlp = GateUpDownMLPGguf::new_from_gguf(gguf, prefix)?; + let mlp = GateUpDownMLPGguf::new_from_gguf( + gguf, + prefix, + false, + None, + None, + None, + Some(candle_nn::Activation::Silu), + )?; let input_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_norm.weight"))?; let input_layernorm = Qwen3_5RMSNorm::from_weight(input_norm_weight, rms_norm_eps)?; let post_norm_weight = @@ -857,16 +880,6 @@ impl Qwen3_5DecoderLayer { let residual = xs.clone(); let mut xs = self.input_layernorm.forward(xs)?; xs = self.attn.forward(&xs, cos, sin, attention_mask)?; - // if self.layer_type.eq("linear_attention") - // && let Some(linear_attn) = self.linear_attn.as_mut() - // { - // xs = linear_attn.forward(&xs, attention_mask)?; - // } else if let Some(self_attn) = self.self_attn.as_mut() - // && let Some(cos) = cos - // && let Some(sin) = sin - // { - // xs = self_attn.forward(&xs, cos, sin, attention_mask)?; - // } let residual = xs.add(&residual)?; xs = self.post_attention_layernorm.forward(&residual)?; xs = self.mlp.forward(&xs)?; @@ -920,6 +933,14 @@ impl Qwen3_5TextModel { }) } pub fn new_from_gguf(gguf: &mut Gguf, device: &Device) -> Result { + let dtype = match gguf.get_matedata("general.dtype") { + Ok(v) => match v.to_u32() { + Ok(0) => DType::F32, + Ok(1) => DType::F16, + _ => DType::F16, + }, + Err(_) => DType::F16, + }; let num_layers = gguf.get_matedata("qwen35.block_count")?.to_u32()? as usize; let full_attention_interval = gguf .get_matedata("qwen35.full_attention_interval")? @@ -939,6 +960,10 @@ impl Qwen3_5TextModel { .to_f32()? as f64; let hidden_size = gguf.get_matedata("qwen35.embedding_length")?.to_u32()? as usize; // 1024 let embed_tensor = gguf.tensor("token_embd.weight")?; + // let embed_tokens = match dtype { + // DType::F32 => Embedding::new(embed_tensor.dequantize(device)?, hidden_size), + // _ => Embedding::new(embed_tensor.dequantize_f16(device)?, hidden_size), + // }; let embed_tokens = Embedding::new(embed_tensor.dequantize(device)?, hidden_size); let mut layers = vec![]; for i in 0..num_layers { @@ -956,14 +981,7 @@ impl Qwen3_5TextModel { let norm_weight = gguf.get_dequantized("output_norm.weight")?; let norm = Qwen3_5RMSNorm::from_weight(norm_weight, rms_norm_eps)?; let rotary_emb = Qwen3VLTextRotaryEmbedding::new(rope_dimension_count, rope_freq_base); - let dtype = match gguf.get_matedata("general.type") { - Ok(v) => match v.to_u32() { - Ok(0) => DType::F32, - Ok(1) => DType::F16, - _ => DType::F16, - }, - Err(_) => DType::F16, - }; + Ok(Self { embed_tokens, layers, @@ -993,6 +1011,7 @@ impl Qwen3_5TextModel { )?) } }; + // let mut i = 0; for layer in self.layers.iter_mut() { let layer_mask = if layer.layer_type.ne("linear_attention") || (seq_len != 1 && b_size != 1) { @@ -1001,8 +1020,11 @@ impl Qwen3_5TextModel { None }; xs = layer.forward(&xs, Some(&cos), Some(&sin), layer_mask.as_ref())?; + // println!("layer {i} : {}", xs); + // i += 1; } xs = self.norm.forward(&xs)?; + // println!("norm : {}", xs); Ok(xs) } @@ -1052,11 +1074,21 @@ impl Qwen3_5Model { }) } - pub fn new_from_gguf(gguf: &mut Gguf, device: &Device) -> Result { + pub fn new_from_gguf( + gguf: &mut Gguf, + mmproj_gguf: Option<&mut Gguf>, + device: &Device, + ) -> Result { let spatial_merge_size = 2usize; let image_token_id = 248056u32; let video_token_id = 248057u32; let vision_start_token_id = 248053u32; + let visual = if let Some(mmproj) = mmproj_gguf { + let visual = Qwen3VLVisionModel::new_from_gguf(mmproj)?; + Some(visual) + } else { + None + }; let language_model = Qwen3_5TextModel::new_from_gguf(gguf, device)?; let lm_head_tensor = match gguf.tensor("output.weight") { Ok(tensor) => tensor, @@ -1068,9 +1100,9 @@ impl Qwen3_5Model { image_token_id, video_token_id, vision_start_token_id, - visual: None, + visual, language_model, - lm_head: ProjKind::QuantizedProj(lm_head), + lm_head: ProjKind::QuantizedProj(QuantizedLinear::new(lm_head, None)), rope_deltas: None, }) } @@ -1351,6 +1383,7 @@ impl Qwen3_5Model { seqlen_offset: usize, ) -> Result { let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?; + // println!("embed_tokens: {}", inputs_embeds); if let Some(pixel_values) = pixel_values && let Some(image_grid_thw) = image_grid_thw && let Some(visual) = self.visual.as_ref() @@ -1365,6 +1398,7 @@ impl Qwen3_5Model { image_embeds.dim(0)? ))); } + let image_embeds = image_embeds.to_dtype(inputs_embeds.dtype())?; inputs_embeds = masked_scatter_dim0(&inputs_embeds, &image_embeds, &vision_mask)?; } if let Some(pixel_values_video) = pixel_values_video @@ -1381,9 +1415,10 @@ impl Qwen3_5Model { video_embeds.dim(0)? ))); } + let video_embeds = video_embeds.to_dtype(inputs_embeds.dtype())?; inputs_embeds = masked_scatter_dim0(&inputs_embeds, &video_embeds, &vision_mask)?; } - + // println!("visual : {}", inputs_embeds); let position_ids = self.compute_3d_position_ids( input_ids, &inputs_embeds, @@ -1394,7 +1429,9 @@ impl Qwen3_5Model { let outputs = self.language_model.forward(&inputs_embeds, &position_ids)?; let seq_len = outputs.dim(1)?; let hidden_state = outputs.narrow(1, seq_len - 1, 1)?; + // println!("narrow 1 : {}", hidden_state); let logits = self.lm_head.forward(&hidden_state)?; + // println!("logits : {}", logits); Ok(logits) } diff --git a/src/models/qwen3vl/model.rs b/src/models/qwen3vl/model.rs index 4a32f90..1113dad 100644 --- a/src/models/qwen3vl/model.rs +++ b/src/models/qwen3vl/model.rs @@ -1,3 +1,5 @@ +use std::io::{Read, Seek}; + use anyhow::{Result, anyhow}; use candle_core::{D, DType, IndexOp, Shape, Tensor}; use candle_nn::{ @@ -7,7 +9,10 @@ use candle_nn::{ use crate::{ models::{ - common::{TwoLinearMLP, eager_attention_forward, get_layer_norm}, + common::{ + eager_attention_forward, get_layer_norm, + gguf::{Gguf, ProjKind, TwoLinearMLPGguf}, + }, qwen3::model::Qwen3DecoderLayer, qwen3vl::config::{ Qwen3VLConfig, Qwen3VLTextConfig, Qwen3VLVisionConfig, qwen3vl_text_config2qwen3_config, @@ -60,6 +65,33 @@ impl Qwen3VLVisionPatchEmbed { }) } + pub fn new_from_gguf(gguf: &mut Gguf) -> Result { + // convert_hf_to_gguf.py + // temporal_patch_size = 2 + // spilt (embed_dim, in_channels, temporal_patch_size, patch_size, patch_size) + // into two (embed_dim, in_channels, patch_size, patch_size) + // elif 'patch_embed.proj.weight' in name: + // # split Conv3D into Conv2Ds + // c1, c2, kt, kh, kw = data_torch.shape + // del c1, c2, kh, kw # unused + // assert kt == 2, "Current implementation only support temporal_patch_size of 2" + // yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight" , data_torch[:, :, 0, ...]) + // yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...]) + // (embed_dim, in_channels, patch_size, patch_size) -> (embed_dim, in_channels, 1, patch_size, patch_size) + let conv3d_weight_0 = gguf.get_dequantized("v.patch_embd.weight")?.unsqueeze(2)?; + let conv3d_weight_1 = gguf + .get_dequantized("v.patch_embd.weight.1")? + .unsqueeze(2)?; + let conv3d_weight = Tensor::cat(&[conv3d_weight_0, conv3d_weight_1], 2)? + .flatten(1, 4)? + .t()?; + let conv3d_bias = gguf.get_dequantized("v.patch_embd.bias")?; + Ok(Self { + conv3d_weight, + conv3d_bias, + }) + } + pub fn forward(&self, hidden_states: &Tensor) -> Result { // hidden_states shape: (grid_t*grid_h*grid_w, c*temporal_patch_size*patch_size*patch_size) // ((), 1536) matmul (1536, 1024) -> ((), 1024) @@ -73,9 +105,11 @@ pub struct Qwen3VLVisionPatchMerger { hidden_size: usize, use_postshuffle_norm: bool, norm: LayerNorm, - linear_fc1: Linear, + // linear_fc1: Linear, + linear_fc1: ProjKind, act_fn: Activation, - linear_fc2: Linear, + // linear_fc2: Linear, + linear_fc2: ProjKind, } impl Qwen3VLVisionPatchMerger { @@ -98,9 +132,34 @@ impl Qwen3VLVisionPatchMerger { hidden_size, use_postshuffle_norm, norm, - linear_fc1, + linear_fc1: ProjKind::LinearProj(linear_fc1), act_fn, - linear_fc2, + linear_fc2: ProjKind::LinearProj(linear_fc2), + }) + } + + pub fn new_from_gguf( + gguf: &mut Gguf, + rms_norm_eps: f64, + use_postshuffle_norm: bool, + hidden_size: usize, + spatial_merge_size: usize, + norm_prefix: &str, + linear1_prefix: &str, + linear2_prefix: &str, + ) -> Result { + let hidden_size = hidden_size * spatial_merge_size.pow(2); + let norm = gguf.layer_norm(norm_prefix, rms_norm_eps)?; + let linear_1 = gguf.quantize_linear(linear1_prefix, true)?; + let act_fn = Activation::Gelu; + let linear_2 = gguf.quantize_linear(linear2_prefix, true)?; + Ok(Self { + hidden_size, + use_postshuffle_norm, + norm, + linear_fc1: ProjKind::QuantizedProj(linear_1), + act_fn, + linear_fc2: ProjKind::QuantizedProj(linear_2), }) } @@ -120,8 +179,10 @@ impl Qwen3VLVisionPatchMerger { pub struct Qwen3VLVisionAttention { num_heads: usize, - qkv: Linear, - proj: Linear, + // qkv: Linear, + // proj: Linear, + qkv: ProjKind, + proj: ProjKind, scaling: f64, } @@ -136,8 +197,27 @@ impl Qwen3VLVisionAttention { Ok(Self { num_heads, - qkv, - proj, + qkv: ProjKind::LinearProj(qkv), + proj: ProjKind::LinearProj(proj), + scaling, + }) + } + + pub fn new_from_gguf(mmproj_gguf: &mut Gguf, prefix: &str) -> Result { + let num_heads = mmproj_gguf + .get_matedata("clip.vision.attention.head_count")? + .to_u32()? as usize; + let hidden_size = mmproj_gguf + .get_matedata("clip.vision.embedding_length")? + .to_u32()? as usize; + let head_dim = hidden_size / num_heads; + let scaling = 1.0 / (head_dim as f64).sqrt(); + let qkv = mmproj_gguf.quantize_linear(&format!("{prefix}.attn_qkv"), true)?; + let proj = mmproj_gguf.quantize_linear(&format!("{prefix}.attn_out"), true)?; + Ok(Self { + num_heads, + qkv: ProjKind::QuantizedProj(qkv), + proj: ProjKind::QuantizedProj(proj), scaling, }) } @@ -195,7 +275,8 @@ pub struct Qwen3VLVisionBlock { norm1: LayerNorm, norm2: LayerNorm, attn: Qwen3VLVisionAttention, - mlp: TwoLinearMLP, + // mlp: TwoLinearMLP, + mlp: TwoLinearMLPGguf, } impl Qwen3VLVisionBlock { @@ -203,7 +284,17 @@ impl Qwen3VLVisionBlock { let norm1 = get_layer_norm(vb.pp("norm1"), 1e-6, config.hidden_size, true)?; let norm2 = get_layer_norm(vb.pp("norm2"), 1e-6, config.hidden_size, true)?; let attn = Qwen3VLVisionAttention::new(config.clone(), vb.pp("attn"))?; - let mlp = TwoLinearMLP::new( + // let mlp = TwoLinearMLP::new( + // vb.pp("mlp"), + // config.hidden_size, + // config.intermediate_size, + // config.hidden_size, + // config.hidden_act, + // true, + // "linear_fc1", + // "linear_fc2", + // )?; + let mlp = TwoLinearMLPGguf::new( vb.pp("mlp"), config.hidden_size, config.intermediate_size, @@ -221,6 +312,30 @@ impl Qwen3VLVisionBlock { }) } + pub fn new_from_gguf( + mmproj_gguf: &mut Gguf, + prefix: &str, + rms_norm_eps: f64, + ) -> Result { + let norm1 = mmproj_gguf.layer_norm(&format!("{prefix}.ln1"), rms_norm_eps)?; + let norm2 = mmproj_gguf.layer_norm(&format!("{prefix}.ln2"), rms_norm_eps)?; + let attn = Qwen3VLVisionAttention::new_from_gguf(mmproj_gguf, prefix)?; + let mlp = TwoLinearMLPGguf::new_from_gguf( + mmproj_gguf, + prefix, + true, + Some("ffn_up"), + Some("ffn_down"), + Some(Activation::GeluPytorchTanh), + )?; + Ok(Self { + norm1, + norm2, + attn, + mlp, + }) + } + pub fn forward( &self, xs: &Tensor, @@ -292,6 +407,92 @@ impl Qwen3VLVisionModel { }) } + pub fn new_from_gguf(mmproj_gguf: &mut Gguf) -> Result { + // let num_layers = gguf.get_matedata("qwen35.block_count")?.to_u32()? as usize; + let spatial_merge_size = mmproj_gguf + .get_matedata("clip.vision.spatial_merge_size")? + .to_u32()? as usize; + let patch_embed = Qwen3VLVisionPatchEmbed::new_from_gguf(mmproj_gguf)?; + let pos_emb_weight = mmproj_gguf.get_dequantized("v.position_embd.weight")?; + let hidden_size = mmproj_gguf + .get_matedata("clip.vision.embedding_length")? + .to_u32()? as usize; + let pos_embed = Embedding::new(pos_emb_weight, hidden_size); + let patch_size = mmproj_gguf + .get_matedata("clip.vision.patch_size")? + .to_u32()? as usize; + let image_size = mmproj_gguf + .get_matedata("clip.vision.image_size")? + .to_u32()? as usize; + let num_grid_per_side = image_size / patch_size; + let num_heads = mmproj_gguf + .get_matedata("clip.vision.attention.head_count")? + .to_u32()? as usize; + let head_dim = hidden_size / num_heads; + let rotary_pos_emb = Qwen2_5VisionRotaryEmbedding::new(head_dim / 2, None); + let rms_norm_eps = mmproj_gguf + .get_matedata("clip.vision.attention.layer_norm_epsilon")? + .to_f32()? as f64; + let mut blocks = Vec::new(); + let num_block = mmproj_gguf + .get_matedata("clip.vision.block_count")? + .to_u32()? as usize; + for i in 0..num_block { + let prefix = format!("v.blk.{i}"); + // let block = Qwen3VLVisionBlock::new(config.clone(), vb_blocks.pp(i))?; + let block = Qwen3VLVisionBlock::new_from_gguf(mmproj_gguf, &prefix, rms_norm_eps)?; + blocks.push(block); + } + let merger = Qwen3VLVisionPatchMerger::new_from_gguf( + mmproj_gguf, + rms_norm_eps, + false, + hidden_size, + spatial_merge_size, + "v.post_ln", + "mm.0", + "mm.2", + )?; + let mut deepstack_merger_list = Vec::new(); + let is_deepstack = mmproj_gguf + .get_matedata("clip.vision.is_deepstack_layers")? + .to_vec()? + .iter() + .map(|b| b.to_bool()) + .collect::, candle_core::Error>>()?; + let deepstack_visual_indexes = is_deepstack + .iter() + .enumerate() + .filter_map(|(i, &b)| if b { Some(i) } else { None }) + .collect::>(); + for i in &deepstack_visual_indexes { + let prefix = format!("v.deepstack.{i}"); + let merger_i = Qwen3VLVisionPatchMerger::new_from_gguf( + mmproj_gguf, + rms_norm_eps, + true, + hidden_size, + spatial_merge_size, + &format!("{prefix}.norm"), + &format!("{prefix}.fc1"), + &format!("{prefix}.fc2"), + )?; + deepstack_merger_list.push(merger_i); + } + Ok(Self { + spatial_merge_size, + patch_embed, + pos_embed, + num_grid_per_side: num_grid_per_side as u32, + rotary_pos_emb, + blocks, + merger, + deepstack_visual_indexes, + deepstack_merger_list, + dtype: DType::F32, + }) + } + pub fn fast_pos_embed_interpolate(&self, grid_thw: &Tensor) -> Result { let mut idx_list = vec![vec![]; 4]; let mut weight_list = vec![vec![]; 4]; diff --git a/src/models/qwen3vl/processor.rs b/src/models/qwen3vl/processor.rs index 46c95a3..81eb2a7 100644 --- a/src/models/qwen3vl/processor.rs +++ b/src/models/qwen3vl/processor.rs @@ -101,7 +101,6 @@ impl Qwen3VLProcessor { max_frames: 768, }) } - pub fn new_qwen3_5_default(device: &Device, dtype: DType) -> Result { let img_process_cfg = PreprocessorConfig::qwen3_5_img_default(); let video_process_cfg = PreprocessorConfig::qwen3_5_video_default(); diff --git a/tests/test_gguf_qwen3_5.rs b/tests/test_gguf_qwen3_5.rs index 85c9946..c7b3c2c 100644 --- a/tests/test_gguf_qwen3_5.rs +++ b/tests/test_gguf_qwen3_5.rs @@ -5,24 +5,40 @@ use aha::{ models::{GenerateModel, qwen3_5::generate::Qwen3_5GenerateModel}, }; use anyhow::Result; -// use candle_core::{Device, quantized::gguf_file}; +// use candle_core::{DType, Device, quantized::gguf_file}; #[test] fn gguf_test() -> Result<()> { - // cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture - // let path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题 - // let path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf"; - let path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf"; - // let mut file = std::fs::File::open(path)?; - // let model = gguf_file::Content::read(&mut file)?; - // println!("group_count: {:?}", model.metadata.get("qwen35.ssm.group_count")); - // println!("time_step_rank: {:?}", model.metadata.get("qwen35.ssm.time_step_rank")); - // println!("state_size: {:?}", model.metadata.get("qwen35.ssm.state_size")); + // RUST_BACKTRACE=1 cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture + // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题 + // let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/mmproj-F16.gguf"; + // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf"; + let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf"; + let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/mmproj-F16.gguf"; + // let mut model_file = std::fs::File::open(model_path)?; + // let model = gguf_file::Content::read(&mut model_file)?; // for (key, value) in model.metadata { - // if key.contains("tokenizer") { + // if key.contains("tokeni") { // continue; // } // println!("{key}: {:#?}", value); // } + // let mut mmproj_file = std::fs::File::open(mmproj_path)?; + // let mmproj = gguf_file::Content::read(&mut mmproj_file)?; + // println!("model: {:#?}", mmproj.tensor_infos.keys()); + // println!("group_count: {:?}", model.metadata.get("qwen35.ssm.group_count")); + // println!("time_step_rank: {:?}", model.metadata.get("qwen35.ssm.time_step_rank")); + // println!("state_size: {:?}", model.metadata.get("qwen35.ssm.state_size")); + // for (key, value) in mmproj.metadata { + // println!("{key}: {:#?}", value); + // } + // let device = Device::new_cuda(0)?; + // let mut mmproj_gguf = Gguf::new(mmproj, mmproj_file, device.clone()); + // let weight = mmproj_gguf.get_dequantized("v.position_embd.weight")?; + // println!("weight: {:?}", weight); + // let conv3d_weight_1 = mmproj_gguf.get_dequantized("v.patch_embd.weight.1")?; + // println!("conv3d_weight_1: {}", conv3d_weight_1); + // let conv3d_bias = mmproj_gguf.get_dequantized("v.patch_embd.bias")?; + // println!("conv3d_bias: {}", conv3d_bias); // println!("model: {:?}", model.magic); // println!("generat.type: {:#?}", model.metadata.keys()); // println!("tokenizer.ggml.eos_token_id: {:#?}", model.metadata.get("tokenizer.ggml.eos_token_id")); @@ -33,19 +49,28 @@ fn gguf_test() -> Result<()> { "messages": [ { "role": "user", - "content": [ + "content": [ { - "type": "text", - "text": "你如何看待AI" + "type": "image", + "image_url": + { + "url": "file://./assets/img/ocr_test1.png" + } + }, + { + "type": "text", + "text": "请分析图片并提取所有可见文本内容，按从左到右、从上到下的布局，返回纯文本" } ] } ] } "#; + let mes: ChatCompletionParameters = serde_json::from_str(message)?; let i_start = Instant::now(); - let mut gguf_qwen3_5 = Qwen3_5GenerateModel::init_from_gguf(path, None, None)?; + let mut gguf_qwen3_5 = + Qwen3_5GenerateModel::init_from_gguf(model_path, mmproj_path.into(), None)?; let i_duration = i_start.elapsed(); println!("Time elapsed in load model is: {:?}", i_duration); diff --git a/tests/test_qwen3_5.rs b/tests/test_qwen3_5.rs index f333c43..f515036 100644 --- a/tests/test_qwen3_5.rs +++ b/tests/test_qwen3_5.rs @@ -24,12 +24,12 @@ fn qwen3_5_generate() -> Result<()> { "type": "image", "image_url": { - "url": "file:///home/jhq/Downloads/gougou1.jpg" + "url": "file://./assets/img/ocr_test1.png" } }, { "type": "text", - "text": "描述这张图片." + "text": "OCR" } ] } diff --git a/tests/test_qwen3vl.rs b/tests/test_qwen3vl.rs index 22915a3..a431b87 100644 --- a/tests/test_qwen3vl.rs +++ b/tests/test_qwen3vl.rs @@ -29,7 +29,7 @@ fn qwen3vl_thinking_generate() -> Result<()> { }, { "type": "text", - "text": "请分析图片并提取所有可见文本内容，按从左到右、从上到下的布局，返回纯文本" + "text": "OCR" } ] } @@ -59,7 +59,7 @@ fn qwen3vl_thinking_generate() -> Result<()> { #[test] fn qwen3vl_generate() -> Result<()> { - // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda,ffmpeg --test test_qwen3vl qwen3vl_generate -r -- --nocapture + // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_qwen3vl qwen3vl_generate -r -- --nocapture let save_dir = aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; @@ -73,15 +73,15 @@ fn qwen3vl_generate() -> Result<()> { "role": "user", "content": [ { - "type": "video", - "video_url": + "type": "image", + "image_url": { - "url": "./assets/video/video_test.mp4" + "url": "file://./assets/img/ocr_test1.png" } - }, + }, { "type": "text", - "text": "视频中发生了什么？" + "text": "OCR" } ] }