tckm.yan/aha
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add Qwen3.5 mmproj gguf

Browse Source
This commit is contained in:
jhqxxx
2026-03-16 14:50:59 +08:00
parent 9c9275b767
commit 80d36b5308
14 changed files with 575 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/common/gguf.rs
+168 -15
View File
@@ -3,13 +3,15 @@ use std::io::{Read, Seek};
use ahash::AHashMap;
use anyhow::{Result, anyhow};
use candle_core::{
Device, Tensor,
DType, Device, Tensor,
quantized::{
QMatMul, QTensor,
gguf_file::{self, Value},
},
};
use candle_nn::{Conv1d, Conv1dConfig, Linear, Module, RmsNorm, VarBuilder, linear_b};
use candle_nn::{
Activation, Conv1d, Conv1dConfig, LayerNorm, Linear, Module, RmsNorm, VarBuilder, linear_b,
};
use tokenizers::{self, AddedToken, Tokenizer, models::bpe::BPE};
use crate::tokenizer::TokenizerModel;
@@ -37,12 +39,40 @@ impl<R: Read + Seek> Gguf<R> {
Ok(QMatMul::from_qtensor(ws)?)
}
pub fn quantize_linear(&mut self, prefix: &str, bias: bool) -> Result<QuantizedLinear> {
let weight = self.qmatmul(&format!("{prefix}.weight"))?;
let bias = if bias {
self.get_dequantized(&format!("{prefix}.bias")).ok()
} else {
None
};
Ok(QuantizedLinear::new(weight, bias))
}
pub fn rms_norm(&mut self, name: &str, eps: f64) -> Result<RmsNorm> {
let ws = self.ct.tensor(&mut self.reader, name, &self.device)?;
let weight = ws.dequantize(&self.device)?;
Ok(RmsNorm::new(weight, eps))
}
pub fn layer_norm(&mut self, prefix: &str, eps: f64) -> Result<LayerNorm> {
let weight = self
.ct
.tensor(&mut self.reader, &format!("{prefix}.weight"), &self.device)?;
let weight = weight.dequantize(&self.device)?;
let bias = self
.ct
.tensor(&mut self.reader, &format!("{prefix}.bias"), &self.device);
let bias = match bias {
Ok(bias) => bias.dequantize(&self.device).ok(),
Err(_) => None,
};
match bias {
Some(bias) => Ok(LayerNorm::new(weight, bias, eps)),
None => Ok(LayerNorm::new_no_bias(weight, eps)),
}
}
pub fn metadata(&self) -> &std::collections::HashMap<String, gguf_file::Value> {
&self.ct.metadata
}
@@ -55,6 +85,10 @@ impl<R: Read + Seek> Gguf<R> {
Ok(self.tensor(name)?.dequantize(&self.device)?)
}
pub fn get_dequantized_f16(&mut self, name: &str) -> Result<Tensor> {
Ok(self.tensor(name)?.dequantize_f16(&self.device)?)
}
pub fn conv1d(
&mut self,
prefix: &str,
@@ -169,9 +203,36 @@ impl<R: Read + Seek> Gguf<R> {
}
}
#[derive(Debug, Clone)]
pub struct QuantizedLinear {
inner: QMatMul,
bias: Option<Tensor>,
}
impl QuantizedLinear {
pub fn new(inner: QMatMul, bias: Option<Tensor>) -> Self {
Self { inner, bias }
}
}
impl Module for QuantizedLinear {
fn forward(&self, xs: &Tensor) -> candle_core::Result<Tensor> {
let xs = if xs.dtype() == DType::F16 {
self.inner.forward_via_f16(xs)?
} else {
self.inner.forward(xs)?
};
if let Some(bias) = &self.bias {
xs.broadcast_add(&bias.to_dtype(xs.dtype())?)
} else {
Ok(xs)
}
}
}
#[derive(Debug, Clone)]
pub enum ProjKind {
QuantizedProj(QMatMul),
QuantizedProj(QuantizedLinear),
LinearProj(Linear),
}
@@ -186,20 +247,52 @@ impl Module for ProjKind {
#[derive(Debug, Clone)]
pub struct GateUpDownMLPGguf {
gate_proj: ProjKind, // ffn_gate.weight
up_proj: ProjKind, // ffn_up.weight
down_proj: ProjKind, // ffn_down.weight
gate_proj: ProjKind,
up_proj: ProjKind,
down_proj: ProjKind,
act: Activation,
}
impl GateUpDownMLPGguf {
pub fn new_from_gguf<R: Read + Seek>(gguf: &mut Gguf<R>, prefix: &str) -> Result<Self> {
let gate_proj = gguf.qmatmul(&format!("{prefix}.ffn_gate.weight"))?;
let up_proj = gguf.qmatmul(&format!("{prefix}.ffn_up.weight"))?;
let down_proj = gguf.qmatmul(&format!("{prefix}.ffn_down.weight"))?;
pub fn new_from_gguf<R: Read + Seek>(
gguf: &mut Gguf<R>,
prefix: &str,
bias: bool,
gate_name: Option<&str>,
up_name: Option<&str>,
down_name: Option<&str>,
act: Option<Activation>,
) -> Result<Self> {
let gate_name = gate_name.unwrap_or("ffn_gate");
let up_name = up_name.unwrap_or("ffn_up");
let down_name = down_name.unwrap_or("ffn_down");
let gate_proj = gguf.qmatmul(&format!("{prefix}.{gate_name}.weight"))?;
let gate_bias = if bias {
gguf.get_dequantized(&format!("{prefix}.{gate_name}.bias"))
.ok()
} else {
None
};
let up_proj = gguf.qmatmul(&format!("{prefix}.{up_name}.weight"))?;
let up_bias = if bias {
gguf.get_dequantized(&format!("{prefix}.{up_name}.bias"))
.ok()
} else {
None
};
let down_proj = gguf.qmatmul(&format!("{prefix}.{down_name}.weight"))?;
let down_bias = if bias {
gguf.get_dequantized(&format!("{prefix}.{down_name}.bias"))
.ok()
} else {
None
};
let act = act.unwrap_or(Activation::Silu);
Ok(Self {
gate_proj: ProjKind::QuantizedProj(gate_proj),
up_proj: ProjKind::QuantizedProj(up_proj),
down_proj: ProjKind::QuantizedProj(down_proj),
gate_proj: ProjKind::QuantizedProj(QuantizedLinear::new(gate_proj, gate_bias)),
up_proj: ProjKind::QuantizedProj(QuantizedLinear::new(up_proj, up_bias)),
down_proj: ProjKind::QuantizedProj(QuantizedLinear::new(down_proj, down_bias)),
act,
})
}
pub fn new_from_vb(
@@ -210,6 +303,7 @@ impl GateUpDownMLPGguf {
gate_pp_name: Option<&str>,
up_pp_name: Option<&str>,
down_pp_name: Option<&str>,
act: Option<Activation>,
) -> Result<Self> {
let gate_pp_name = gate_pp_name.unwrap_or("gate_proj");
let up_pp_name = up_pp_name.unwrap_or("up_proj");
@@ -217,18 +311,77 @@ impl GateUpDownMLPGguf {
let gate_proj = linear_b(hidden_size, intermediate_size, bias, vb.pp(gate_pp_name))?;
let up_proj = linear_b(hidden_size, intermediate_size, bias, vb.pp(up_pp_name))?;
let down_proj = linear_b(intermediate_size, hidden_size, bias, vb.pp(down_pp_name))?;
let act = act.unwrap_or(Activation::Silu);
Ok(Self {
gate_proj: ProjKind::LinearProj(gate_proj),
up_proj: ProjKind::LinearProj(up_proj),
down_proj: ProjKind::LinearProj(down_proj),
act,
})
}
}
impl Module for GateUpDownMLPGguf {
fn forward(&self, xs: &Tensor) -> candle_core::Result<Tensor> {
let w1 = self.gate_proj.forward(xs)?;
let w1 = self.gate_proj.forward(xs)?.apply(&self.act)?;
let w3 = self.up_proj.forward(xs)?;
self.down_proj.forward(&(candle_nn::ops::silu(&w1)? * w3)?)
self.down_proj.forward(&(w1 * w3)?)
}
}
pub struct TwoLinearMLPGguf {
linear1: ProjKind,
linear2: ProjKind,
act: Activation,
}
impl TwoLinearMLPGguf {
pub fn new(
vb: VarBuilder,
// embedding_dim: usize,
// mlp_dim: usize,
in_dim: usize,
middle_dim: usize,
out_dim: usize,
act: Activation,
bias: bool,
linear1_pp_name: &str,
linear2_pp_name: &str,
) -> Result<Self> {
let linear1 = linear_b(in_dim, middle_dim, bias, vb.pp(linear1_pp_name))?;
let linear2 = linear_b(middle_dim, out_dim, bias, vb.pp(linear2_pp_name))?;
Ok(Self {
linear1: ProjKind::LinearProj(linear1),
linear2: ProjKind::LinearProj(linear2),
act,
})
}
pub fn new_from_gguf<R: Read + Seek>(
gguf: &mut Gguf<R>,
prefix: &str,
bias: bool,
linear1_name: Option<&str>,
linear2_name: Option<&str>,
act: Option<Activation>,
) -> Result<Self> {
let linear1_name = linear1_name.unwrap_or("ffn_up");
let linear2_name = linear2_name.unwrap_or("ffn_down");
let linear1 = gguf.quantize_linear(&format!("{prefix}.{linear1_name}"), bias)?;
let linear2 = gguf.quantize_linear(&format!("{prefix}.{linear2_name}"), bias)?;
let act = act.unwrap_or(Activation::Silu);
Ok(Self {
linear1: ProjKind::QuantizedProj(linear1),
linear2: ProjKind::QuantizedProj(linear2),
act,
})
}
pub fn forward(&self, xs: &Tensor) -> Result<Tensor> {
let xs = xs
.apply(&self.linear1)?
.apply(&self.act)?
.apply(&self.linear2)?;
Ok(xs)
}
}
src/models/common/mod.rs
+2 -2
View File
@@ -1172,7 +1172,7 @@ pub fn conv1d_depthwise(input: &Tensor, weight: &Tensor, bias: Option<&Tensor>)
// weight: (dim, 1, k) -> (dim, k)
// input already padding
let len_in = input.dim(2)?;
let weight = weight.squeeze(1)?;
let weight = weight.squeeze(1)?.to_dtype(input.dtype())?;
let kernel_size = weight.dim(1)?;
// len_out = (len_in - k + 2p) / s + 1, p = 0, s = 1
let len_out = len_in - kernel_size + 1;
@@ -1189,7 +1189,7 @@ pub fn conv1d_depthwise(input: &Tensor, weight: &Tensor, bias: Option<&Tensor>)
None => Ok(out),
Some(bias) => {
let b = bias.dims1()?;
let bias = bias.reshape((1, b, 1))?;
let bias = bias.reshape((1, b, 1))?.to_dtype(input.dtype())?;
Ok(out.broadcast_add(&bias)?)
}
}
src/models/mod.rs
+5 -5
View File
@@ -168,7 +168,7 @@ pub enum ModelInstance<'a> {
Qwen3(Qwen3GenerateModel<'a>),
Qwen3_5(Qwen3_5GenerateModel<'a>),
Qwen3ASR(Qwen3AsrGenerateModel<'a>),
Qwen3VL(Qwen3VLGenerateModel<'a>),
Qwen3VL(Box<Qwen3VLGenerateModel<'a>>),
DeepSeekOCR(DeepseekOCRGenerateModel),
HunyuanOCR(HunyuanOCRGenerateModel<'a>),
PaddleOCRVL(Box<PaddleOCRVLGenerateModel<'a>>),
@@ -273,19 +273,19 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_
}
WhichModel::Qwen3vl2B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(model)
ModelInstance::Qwen3VL(Box::new(model))
}
WhichModel::Qwen3vl4B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(model)
ModelInstance::Qwen3VL(Box::new(model))
}
WhichModel::Qwen3vl8B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(model)
ModelInstance::Qwen3VL(Box::new(model))
}
WhichModel::Qwen3vl32B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(model)
ModelInstance::Qwen3VL(Box::new(model))
}
WhichModel::DeepSeekOCR => {
let model = DeepseekOCRGenerateModel::init(path, None, None)?;
src/models/qwen3_5/generate.rs
+52 -31
View File
@@ -25,7 +25,7 @@ use crate::{
pub struct Qwen3_5GenerateModel<'a> {
chat_template: ChatTemplate<'a>,
tokenizer: TokenizerModel,
pre_processor: Qwen3VLProcessor,
pre_processor: Option<Qwen3VLProcessor>,
qwen3_5: Qwen3_5Model,
device: Device,
eos_token_id: u32,
@@ -57,7 +57,7 @@ impl<'a> Qwen3_5GenerateModel<'a> {
Ok(Self {
chat_template,
tokenizer,
pre_processor,
pre_processor: Some(pre_processor),
qwen3_5,
device,
eos_token_id,
@@ -84,26 +84,26 @@ impl<'a> Qwen3_5GenerateModel<'a> {
let mut reader = std::fs::File::open(model_file)?;
let content = gguf_file::Content::read(&mut reader)?;
let device = get_device(device);
let mut gguf = Gguf::new(content, reader, device.clone());
let mut model_gguf = Gguf::new(content, reader, device.clone());
let chat_template_str = gguf
let chat_template_str = model_gguf
.get_matedata("tokenizer.chat_template")?
.to_string()?
.clone();
let chat_template = ChatTemplate::str_init(&chat_template_str)?;
let tokenizer = gguf.build_tokenizer(Some(false), Some(false), Some(false))?;
let dtype = match gguf.get_matedata("general.type") {
Ok(v) => match v.to_u32() {
Ok(0) => DType::F32,
Ok(1) => DType::F16,
_ => DType::F16,
},
Err(_) => DType::F16,
let tokenizer = model_gguf.build_tokenizer(Some(false), Some(false), Some(false))?;
let (pre_processor, mut mmproj_gguf) = if let Some(mmproj_f) = mmproj_file {
let mut reader = std::fs::File::open(mmproj_f)?;
let content = gguf_file::Content::read(&mut reader)?;
let mmproj_gguf = Gguf::new(content, reader, device.clone());
let processor = Qwen3VLProcessor::new_qwen3_5_default(&device, DType::F32)?;
(Some(processor), Some(mmproj_gguf))
} else {
(None, None)
};
let pre_processor = Qwen3VLProcessor::new_qwen3_5_default(&device, dtype)?;
// let eos_token_id = gguf.get_matedata("tokenizer.ggml.eos_token_id")?.to_u32()?;
let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut gguf, &device)?;
let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut model_gguf, mmproj_gguf.as_mut(), &device)?;
let stem = std::path::Path::new(model_file)
.file_stem() // 获取文件名主干（不含扩展名）
.and_then(|s| s.to_str())
@@ -131,18 +131,28 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
get_logit_processor(temperature.into(), top_p.into(), Some(20), seed);
// let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
let mes_render = self.chat_template.apply_chat_template(&mes)?;
let input = self.pre_processor.process_info(&mes, &mes_render)?;
let mut input_ids = self
.tokenizer
.text_encode(input.replace_text.clone(), &self.device)?;
let (mes_text, pixel_values, image_grid_thw, pixel_values_video, video_grid_thw) =
if let Some(processor) = &self.pre_processor {
let input = processor.process_info(&mes, &mes_render)?;
(
input.replace_text,
input.pixel_values,
input.image_grid_thw,
input.pixel_values_video,
input.video_grid_thw,
)
} else {
(mes_render, None, None, None, None)
};
// let input = self.pre_processor.process_info(&mes, &mes_render)?;
let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?;
let mut seq_len = input_ids.dim(1)?;
let prompt_tokens = seq_len as u32;
let mut seqlen_offset = 0;
let mut pixel_values = input.pixel_values.as_ref();
let image_grid_thw = input.image_grid_thw.as_ref();
let mut pixel_values_video = input.pixel_values_video.as_ref();
let video_grid_thw = input.video_grid_thw.as_ref();
let mut pixel_values = pixel_values.as_ref();
let image_grid_thw = image_grid_thw.as_ref();
let mut pixel_values_video = pixel_values_video.as_ref();
let video_grid_thw = video_grid_thw.as_ref();
let mut generate = Vec::new();
let sample_len = mes.max_tokens.unwrap_or(1024);
for _ in 0..sample_len {
@@ -202,19 +212,30 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
let seed = mes.seed.unwrap_or(34562) as u64;
let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
let mes_render = self.chat_template.apply_chat_template(&mes)?;
let input = self.pre_processor.process_info(&mes, &mes_render)?;
let mut input_ids = self
.tokenizer
.text_encode(input.replace_text.clone(), &self.device)?;
// let input = self.pre_processor.process_info(&mes, &mes_render)?;
let (mes_text, pixel_values, image_grid_thw, pixel_values_video, video_grid_thw) =
if let Some(processor) = &self.pre_processor {
let input = processor.process_info(&mes, &mes_render)?;
(
input.replace_text,
input.pixel_values,
input.image_grid_thw,
input.pixel_values_video,
input.video_grid_thw,
)
} else {
(mes_render, None, None, None, None)
};
let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?;
let mut seq_len = input_ids.dim(1)?;
let mut seqlen_offset = 0;
let sample_len = mes.max_tokens.unwrap_or(1024);
let stream = stream! {
let mut error_tokens = Vec::new();
let mut pixel_values = input.pixel_values.as_ref();
let image_grid_thw = input.image_grid_thw.as_ref();
let mut pixel_values_video = input.pixel_values_video.as_ref();
let video_grid_thw = input.video_grid_thw.as_ref();
let mut pixel_values = pixel_values.as_ref();
let image_grid_thw = image_grid_thw.as_ref();
let mut pixel_values_video = pixel_values_video.as_ref();
let video_grid_thw = video_grid_thw.as_ref();
let mut tool_call_id = None;
let mut tool_call_content = String::new();
let mut generate = Vec::new();
src/models/qwen3_5/model.rs
+75 -38
View File
@@ -11,7 +11,7 @@ use crate::{
models::{
common::{
conv1d_depthwise, eager_attention_forward, get_conv1d,
gguf::{GateUpDownMLPGguf, Gguf, ProjKind},
gguf::{GateUpDownMLPGguf, Gguf, ProjKind, QuantizedLinear},
softplus,
},
qwen3_5::config::{Qwen3_5Config, Qwen3_5TextConfig},
@@ -55,24 +55,29 @@ impl Qwen3_5RMSNorm {
pub struct Qwen3_5RMSNormGated {
norm: RmsNorm,
dtype: DType,
}
impl Qwen3_5RMSNormGated {
pub fn new(vb: VarBuilder, hidden_size: usize, eps: f64) -> Result<Self> {
let dtype = vb.dtype();
let norm = rms_norm(hidden_size, eps, vb)?;
Ok(Self { norm })
Ok(Self { norm, dtype })
}
pub fn from_weight(weight: Tensor, eps: f64) -> Result<Self> {
let dtype = weight.dtype();
let norm = RmsNorm::new(weight, eps);
Ok(Self { norm })
Ok(Self { norm, dtype })
}
pub fn forward(&self, xs: &Tensor, gate: Option<&Tensor>) -> Result<Tensor> {
let mut xs = self.norm.forward(xs)?;
let orig_dtype = xs.dtype();
let mut xs = self.norm.forward(&xs.to_dtype(self.dtype)?)?;
if let Some(gate) = gate {
xs = xs.broadcast_mul(&gate.silu()?)?;
xs = xs.broadcast_mul(&gate.silu()?.to_dtype(xs.dtype())?)?;
}
xs = xs.to_dtype(orig_dtype)?;
Ok(xs)
}
}
@@ -225,11 +230,16 @@ impl Qwen3_5GatedDeltaNet {
let a_log = gguf.get_dequantized(&format!("{prefix}.ssm_a"))?;
let norm_weight = gguf.get_dequantized(&format!("{prefix}.ssm_norm.weight"))?;
let norm = Qwen3_5RMSNormGated::from_weight(norm_weight, rms_norm_eps)?;
let out_proj = gguf.qmatmul(&format!("{prefix}.ssm_out.weight"))?;
let in_proj_qkv = gguf.qmatmul(&format!("{prefix}.attn_qkv.weight"))?;
let in_proj_z = gguf.qmatmul(&format!("{prefix}.attn_gate.weight"))?;
let in_proj_b = gguf.qmatmul(&format!("{prefix}.ssm_beta.weight"))?;
let in_proj_a = gguf.qmatmul(&format!("{prefix}.ssm_alpha.weight"))?;
// let out_proj = gguf.qmatmul(&format!("{prefix}.ssm_out.weight"))?;
// let in_proj_qkv = gguf.qmatmul(&format!("{prefix}.attn_qkv.weight"))?;
// let in_proj_z = gguf.qmatmul(&format!("{prefix}.attn_gate.weight"))?;
// let in_proj_b = gguf.qmatmul(&format!("{prefix}.ssm_beta.weight"))?;
// let in_proj_a = gguf.qmatmul(&format!("{prefix}.ssm_alpha.weight"))?;
let out_proj = gguf.quantize_linear(&format!("{prefix}.ssm_out"), false)?;
let in_proj_qkv = gguf.quantize_linear(&format!("{prefix}.attn_qkv"), false)?;
let in_proj_z = gguf.quantize_linear(&format!("{prefix}.attn_gate"), false)?;
let in_proj_b = gguf.quantize_linear(&format!("{prefix}.ssm_beta"), false)?;
let in_proj_a = gguf.quantize_linear(&format!("{prefix}.ssm_alpha"), false)?;
Ok(Self {
num_v_heads,
@@ -497,7 +507,7 @@ impl Qwen3_5GatedDeltaNet {
pub fn forward(&mut self, xs: &Tensor, attention_mask: Option<&Tensor>) -> Result<Tensor> {
let xs = if let Some(mask) = attention_mask {
xs.broadcast_mul(&mask.unsqueeze(D::Minus1)?)?
xs.broadcast_mul(&mask.unsqueeze(D::Minus1)?.to_dtype(xs.dtype())?)?
} else {
xs.clone()
};
@@ -654,10 +664,14 @@ impl Qwen3_5Attention {
let num_kv_groups = num_attention_heads / num_key_value_heads;
let head_dim = gguf.get_matedata("qwen35.attention.key_length")?.to_u32()? as usize;
let scaling = 1f64 / f64::sqrt(head_dim as f64);
let q_proj = gguf.qmatmul(&format!("{prefix}.attn_q.weight"))?;
let k_proj = gguf.qmatmul(&format!("{prefix}.attn_k.weight"))?;
let v_proj = gguf.qmatmul(&format!("{prefix}.attn_v.weight"))?;
let o_proj = gguf.qmatmul(&format!("{prefix}.attn_output.weight"))?;
// let q_proj = gguf.qmatmul(&format!("{prefix}.attn_q.weight"))?;
// let k_proj = gguf.qmatmul(&format!("{prefix}.attn_k.weight"))?;
// let v_proj = gguf.qmatmul(&format!("{prefix}.attn_v.weight"))?;
// let o_proj = gguf.qmatmul(&format!("{prefix}.attn_output.weight"))?;
let q_proj = gguf.quantize_linear(&format!("{prefix}.attn_q"), false)?;
let k_proj = gguf.quantize_linear(&format!("{prefix}.attn_k"), false)?;
let v_proj = gguf.quantize_linear(&format!("{prefix}.attn_v"), false)?;
let o_proj = gguf.quantize_linear(&format!("{prefix}.attn_output"), false)?;
let q_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_q_norm.weight"))?;
let q_norm = Qwen3_5RMSNorm::from_weight(q_norm_weight, rms_norm_eps)?;
let k_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_k_norm.weight"))?;
@@ -800,6 +814,7 @@ impl Qwen3_5DecoderLayer {
None,
None,
None,
Some(config.hidden_act),
)?;
let input_layernorm =
Qwen3_5RMSNorm::new(vb.pp("input_layernorm"), hidden_size, config.rms_norm_eps)?;
@@ -831,7 +846,15 @@ impl Qwen3_5DecoderLayer {
let attn = Qwen3_5Attention::new_from_gguf(gguf, prefix, rms_norm_eps)?;
AttnKind::SelfAttn(attn)
};
let mlp = GateUpDownMLPGguf::new_from_gguf(gguf, prefix)?;
let mlp = GateUpDownMLPGguf::new_from_gguf(
gguf,
prefix,
false,
None,
None,
None,
Some(candle_nn::Activation::Silu),
)?;
let input_norm_weight = gguf.get_dequantized(&format!("{prefix}.attn_norm.weight"))?;
let input_layernorm = Qwen3_5RMSNorm::from_weight(input_norm_weight, rms_norm_eps)?;
let post_norm_weight =
@@ -857,16 +880,6 @@ impl Qwen3_5DecoderLayer {
let residual = xs.clone();
let mut xs = self.input_layernorm.forward(xs)?;
xs = self.attn.forward(&xs, cos, sin, attention_mask)?;
// if self.layer_type.eq("linear_attention")
// && let Some(linear_attn) = self.linear_attn.as_mut()
// {
// xs = linear_attn.forward(&xs, attention_mask)?;
// } else if let Some(self_attn) = self.self_attn.as_mut()
// && let Some(cos) = cos
// && let Some(sin) = sin
// {
// xs = self_attn.forward(&xs, cos, sin, attention_mask)?;
// }
let residual = xs.add(&residual)?;
xs = self.post_attention_layernorm.forward(&residual)?;
xs = self.mlp.forward(&xs)?;
@@ -920,6 +933,14 @@ impl Qwen3_5TextModel {
})
}
pub fn new_from_gguf<R: Read + Seek>(gguf: &mut Gguf<R>, device: &Device) -> Result<Self> {
let dtype = match gguf.get_matedata("general.dtype") {
Ok(v) => match v.to_u32() {
Ok(0) => DType::F32,
Ok(1) => DType::F16,
_ => DType::F16,
},
Err(_) => DType::F16,
};
let num_layers = gguf.get_matedata("qwen35.block_count")?.to_u32()? as usize;
let full_attention_interval = gguf
.get_matedata("qwen35.full_attention_interval")?
@@ -939,6 +960,10 @@ impl Qwen3_5TextModel {
.to_f32()? as f64;
let hidden_size = gguf.get_matedata("qwen35.embedding_length")?.to_u32()? as usize; // 1024
let embed_tensor = gguf.tensor("token_embd.weight")?;
// let embed_tokens = match dtype {
// DType::F32 => Embedding::new(embed_tensor.dequantize(device)?, hidden_size),
// _ => Embedding::new(embed_tensor.dequantize_f16(device)?, hidden_size),
// };
let embed_tokens = Embedding::new(embed_tensor.dequantize(device)?, hidden_size);
let mut layers = vec![];
for i in 0..num_layers {
@@ -956,14 +981,7 @@ impl Qwen3_5TextModel {
let norm_weight = gguf.get_dequantized("output_norm.weight")?;
let norm = Qwen3_5RMSNorm::from_weight(norm_weight, rms_norm_eps)?;
let rotary_emb = Qwen3VLTextRotaryEmbedding::new(rope_dimension_count, rope_freq_base);
let dtype = match gguf.get_matedata("general.type") {
Ok(v) => match v.to_u32() {
Ok(0) => DType::F32,
Ok(1) => DType::F16,
_ => DType::F16,
},
Err(_) => DType::F16,
};
Ok(Self {
embed_tokens,
layers,
@@ -993,6 +1011,7 @@ impl Qwen3_5TextModel {
)?)
}
};
// let mut i = 0;
for layer in self.layers.iter_mut() {
let layer_mask =
if layer.layer_type.ne("linear_attention") || (seq_len != 1 && b_size != 1) {
@@ -1001,8 +1020,11 @@ impl Qwen3_5TextModel {
None
};
xs = layer.forward(&xs, Some(&cos), Some(&sin), layer_mask.as_ref())?;
// println!("layer {i} : {}", xs);
// i += 1;
}
xs = self.norm.forward(&xs)?;
// println!("norm : {}", xs);
Ok(xs)
}
@@ -1052,11 +1074,21 @@ impl Qwen3_5Model {
})
}
pub fn new_from_gguf<R: Read + Seek>(gguf: &mut Gguf<R>, device: &Device) -> Result<Self> {
pub fn new_from_gguf<R: Read + Seek>(
gguf: &mut Gguf<R>,
mmproj_gguf: Option<&mut Gguf<R>>,
device: &Device,
) -> Result<Self> {
let spatial_merge_size = 2usize;
let image_token_id = 248056u32;
let video_token_id = 248057u32;
let vision_start_token_id = 248053u32;
let visual = if let Some(mmproj) = mmproj_gguf {
let visual = Qwen3VLVisionModel::new_from_gguf(mmproj)?;
Some(visual)
} else {
None
};
let language_model = Qwen3_5TextModel::new_from_gguf(gguf, device)?;
let lm_head_tensor = match gguf.tensor("output.weight") {
Ok(tensor) => tensor,
@@ -1068,9 +1100,9 @@ impl Qwen3_5Model {
image_token_id,
video_token_id,
vision_start_token_id,
visual: None,
visual,
language_model,
lm_head: ProjKind::QuantizedProj(lm_head),
lm_head: ProjKind::QuantizedProj(QuantizedLinear::new(lm_head, None)),
rope_deltas: None,
})
}
@@ -1351,6 +1383,7 @@ impl Qwen3_5Model {
seqlen_offset: usize,
) -> Result<Tensor> {
let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?;
// println!("embed_tokens: {}", inputs_embeds);
if let Some(pixel_values) = pixel_values
&& let Some(image_grid_thw) = image_grid_thw
&& let Some(visual) = self.visual.as_ref()
@@ -1365,6 +1398,7 @@ impl Qwen3_5Model {
image_embeds.dim(0)?
)));
}
let image_embeds = image_embeds.to_dtype(inputs_embeds.dtype())?;
inputs_embeds = masked_scatter_dim0(&inputs_embeds, &image_embeds, &vision_mask)?;
}
if let Some(pixel_values_video) = pixel_values_video
@@ -1381,9 +1415,10 @@ impl Qwen3_5Model {
video_embeds.dim(0)?
)));
}
let video_embeds = video_embeds.to_dtype(inputs_embeds.dtype())?;
inputs_embeds = masked_scatter_dim0(&inputs_embeds, &video_embeds, &vision_mask)?;
}
// println!("visual : {}", inputs_embeds);
let position_ids = self.compute_3d_position_ids(
input_ids,
&inputs_embeds,
@@ -1394,7 +1429,9 @@ impl Qwen3_5Model {
let outputs = self.language_model.forward(&inputs_embeds, &position_ids)?;
let seq_len = outputs.dim(1)?;
let hidden_state = outputs.narrow(1, seq_len - 1, 1)?;
// println!("narrow 1 : {}", hidden_state);
let logits = self.lm_head.forward(&hidden_state)?;
// println!("logits : {}", logits);
Ok(logits)
}
src/models/qwen3vl/model.rs
+212 -11
View File
@@ -1,3 +1,5 @@
use std::io::{Read, Seek};
use anyhow::{Result, anyhow};
use candle_core::{D, DType, IndexOp, Shape, Tensor};
use candle_nn::{
@@ -7,7 +9,10 @@ use candle_nn::{
use crate::{
models::{
common::{TwoLinearMLP, eager_attention_forward, get_layer_norm},
common::{
eager_attention_forward, get_layer_norm,
gguf::{Gguf, ProjKind, TwoLinearMLPGguf},
},
qwen3::model::Qwen3DecoderLayer,
qwen3vl::config::{
Qwen3VLConfig, Qwen3VLTextConfig, Qwen3VLVisionConfig, qwen3vl_text_config2qwen3_config,
@@ -60,6 +65,33 @@ impl Qwen3VLVisionPatchEmbed {
})
}
pub fn new_from_gguf<R: Read + Seek>(gguf: &mut Gguf<R>) -> Result<Self> {
// convert_hf_to_gguf.py
// temporal_patch_size = 2
// spilt (embed_dim, in_channels, temporal_patch_size, patch_size, patch_size)
// into two (embed_dim, in_channels, patch_size, patch_size)
// elif 'patch_embed.proj.weight' in name:
// # split Conv3D into Conv2Ds
// c1, c2, kt, kh, kw = data_torch.shape
// del c1, c2, kh, kw # unused
// assert kt == 2, "Current implementation only support temporal_patch_size of 2"
// yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight" , data_torch[:, :, 0, ...])
// yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...])
// (embed_dim, in_channels, patch_size, patch_size) -> (embed_dim, in_channels, 1, patch_size, patch_size)
let conv3d_weight_0 = gguf.get_dequantized("v.patch_embd.weight")?.unsqueeze(2)?;
let conv3d_weight_1 = gguf
.get_dequantized("v.patch_embd.weight.1")?
.unsqueeze(2)?;
let conv3d_weight = Tensor::cat(&[conv3d_weight_0, conv3d_weight_1], 2)?
.flatten(1, 4)?
.t()?;
let conv3d_bias = gguf.get_dequantized("v.patch_embd.bias")?;
Ok(Self {
conv3d_weight,
conv3d_bias,
})
}
pub fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
// hidden_states shape: (grid_t*grid_h*grid_w, c*temporal_patch_size*patch_size*patch_size)
// ((), 1536) matmul (1536, 1024) -> ((), 1024)
@@ -73,9 +105,11 @@ pub struct Qwen3VLVisionPatchMerger {
hidden_size: usize,
use_postshuffle_norm: bool,
norm: LayerNorm,
linear_fc1: Linear,
// linear_fc1: Linear,
linear_fc1: ProjKind,
act_fn: Activation,
linear_fc2: Linear,
// linear_fc2: Linear,
linear_fc2: ProjKind,
}
impl Qwen3VLVisionPatchMerger {
@@ -98,9 +132,34 @@ impl Qwen3VLVisionPatchMerger {
hidden_size,
use_postshuffle_norm,
norm,
linear_fc1,
linear_fc1: ProjKind::LinearProj(linear_fc1),
act_fn,
linear_fc2,
linear_fc2: ProjKind::LinearProj(linear_fc2),
})
}
pub fn new_from_gguf<R: Read + Seek>(
gguf: &mut Gguf<R>,
rms_norm_eps: f64,
use_postshuffle_norm: bool,
hidden_size: usize,
spatial_merge_size: usize,
norm_prefix: &str,
linear1_prefix: &str,
linear2_prefix: &str,
) -> Result<Self> {
let hidden_size = hidden_size * spatial_merge_size.pow(2);
let norm = gguf.layer_norm(norm_prefix, rms_norm_eps)?;
let linear_1 = gguf.quantize_linear(linear1_prefix, true)?;
let act_fn = Activation::Gelu;
let linear_2 = gguf.quantize_linear(linear2_prefix, true)?;
Ok(Self {
hidden_size,
use_postshuffle_norm,
norm,
linear_fc1: ProjKind::QuantizedProj(linear_1),
act_fn,
linear_fc2: ProjKind::QuantizedProj(linear_2),
})
}
@@ -120,8 +179,10 @@ impl Qwen3VLVisionPatchMerger {
pub struct Qwen3VLVisionAttention {
num_heads: usize,
qkv: Linear,
proj: Linear,
// qkv: Linear,
// proj: Linear,
qkv: ProjKind,
proj: ProjKind,
scaling: f64,
}
@@ -136,8 +197,27 @@ impl Qwen3VLVisionAttention {
Ok(Self {
num_heads,
qkv,
proj,
qkv: ProjKind::LinearProj(qkv),
proj: ProjKind::LinearProj(proj),
scaling,
})
}
pub fn new_from_gguf<R: Read + Seek>(mmproj_gguf: &mut Gguf<R>, prefix: &str) -> Result<Self> {
let num_heads = mmproj_gguf
.get_matedata("clip.vision.attention.head_count")?
.to_u32()? as usize;
let hidden_size = mmproj_gguf
.get_matedata("clip.vision.embedding_length")?
.to_u32()? as usize;
let head_dim = hidden_size / num_heads;
let scaling = 1.0 / (head_dim as f64).sqrt();
let qkv = mmproj_gguf.quantize_linear(&format!("{prefix}.attn_qkv"), true)?;
let proj = mmproj_gguf.quantize_linear(&format!("{prefix}.attn_out"), true)?;
Ok(Self {
num_heads,
qkv: ProjKind::QuantizedProj(qkv),
proj: ProjKind::QuantizedProj(proj),
scaling,
})
}
@@ -195,7 +275,8 @@ pub struct Qwen3VLVisionBlock {
norm1: LayerNorm,
norm2: LayerNorm,
attn: Qwen3VLVisionAttention,
mlp: TwoLinearMLP,
// mlp: TwoLinearMLP,
mlp: TwoLinearMLPGguf,
}
impl Qwen3VLVisionBlock {
@@ -203,7 +284,17 @@ impl Qwen3VLVisionBlock {
let norm1 = get_layer_norm(vb.pp("norm1"), 1e-6, config.hidden_size, true)?;
let norm2 = get_layer_norm(vb.pp("norm2"), 1e-6, config.hidden_size, true)?;
let attn = Qwen3VLVisionAttention::new(config.clone(), vb.pp("attn"))?;
let mlp = TwoLinearMLP::new(
// let mlp = TwoLinearMLP::new(
// vb.pp("mlp"),
// config.hidden_size,
// config.intermediate_size,
// config.hidden_size,
// config.hidden_act,
// true,
// "linear_fc1",
// "linear_fc2",
// )?;
let mlp = TwoLinearMLPGguf::new(
vb.pp("mlp"),
config.hidden_size,
config.intermediate_size,
@@ -221,6 +312,30 @@ impl Qwen3VLVisionBlock {
})
}
pub fn new_from_gguf<R: Read + Seek>(
mmproj_gguf: &mut Gguf<R>,
prefix: &str,
rms_norm_eps: f64,
) -> Result<Self> {
let norm1 = mmproj_gguf.layer_norm(&format!("{prefix}.ln1"), rms_norm_eps)?;
let norm2 = mmproj_gguf.layer_norm(&format!("{prefix}.ln2"), rms_norm_eps)?;
let attn = Qwen3VLVisionAttention::new_from_gguf(mmproj_gguf, prefix)?;
let mlp = TwoLinearMLPGguf::new_from_gguf(
mmproj_gguf,
prefix,
true,
Some("ffn_up"),
Some("ffn_down"),
Some(Activation::GeluPytorchTanh),
)?;
Ok(Self {
norm1,
norm2,
attn,
mlp,
})
}
pub fn forward(
&self,
xs: &Tensor,
@@ -292,6 +407,92 @@ impl Qwen3VLVisionModel {
})
}
pub fn new_from_gguf<R: Read + Seek>(mmproj_gguf: &mut Gguf<R>) -> Result<Self> {
// let num_layers = gguf.get_matedata("qwen35.block_count")?.to_u32()? as usize;
let spatial_merge_size = mmproj_gguf
.get_matedata("clip.vision.spatial_merge_size")?
.to_u32()? as usize;
let patch_embed = Qwen3VLVisionPatchEmbed::new_from_gguf(mmproj_gguf)?;
let pos_emb_weight = mmproj_gguf.get_dequantized("v.position_embd.weight")?;
let hidden_size = mmproj_gguf
.get_matedata("clip.vision.embedding_length")?
.to_u32()? as usize;
let pos_embed = Embedding::new(pos_emb_weight, hidden_size);
let patch_size = mmproj_gguf
.get_matedata("clip.vision.patch_size")?
.to_u32()? as usize;
let image_size = mmproj_gguf
.get_matedata("clip.vision.image_size")?
.to_u32()? as usize;
let num_grid_per_side = image_size / patch_size;
let num_heads = mmproj_gguf
.get_matedata("clip.vision.attention.head_count")?
.to_u32()? as usize;
let head_dim = hidden_size / num_heads;
let rotary_pos_emb = Qwen2_5VisionRotaryEmbedding::new(head_dim / 2, None);
let rms_norm_eps = mmproj_gguf
.get_matedata("clip.vision.attention.layer_norm_epsilon")?
.to_f32()? as f64;
let mut blocks = Vec::new();
let num_block = mmproj_gguf
.get_matedata("clip.vision.block_count")?
.to_u32()? as usize;
for i in 0..num_block {
let prefix = format!("v.blk.{i}");
// let block = Qwen3VLVisionBlock::new(config.clone(), vb_blocks.pp(i))?;
let block = Qwen3VLVisionBlock::new_from_gguf(mmproj_gguf, &prefix, rms_norm_eps)?;
blocks.push(block);
}
let merger = Qwen3VLVisionPatchMerger::new_from_gguf(
mmproj_gguf,
rms_norm_eps,
false,
hidden_size,
spatial_merge_size,
"v.post_ln",
"mm.0",
"mm.2",
)?;
let mut deepstack_merger_list = Vec::new();
let is_deepstack = mmproj_gguf
.get_matedata("clip.vision.is_deepstack_layers")?
.to_vec()?
.iter()
.map(|b| b.to_bool())
.collect::<Result<Vec<bool>, candle_core::Error>>()?;
let deepstack_visual_indexes = is_deepstack
.iter()
.enumerate()
.filter_map(|(i, &b)| if b { Some(i) } else { None })
.collect::<Vec<usize>>();
for i in &deepstack_visual_indexes {
let prefix = format!("v.deepstack.{i}");
let merger_i = Qwen3VLVisionPatchMerger::new_from_gguf(
mmproj_gguf,
rms_norm_eps,
true,
hidden_size,
spatial_merge_size,
&format!("{prefix}.norm"),
&format!("{prefix}.fc1"),
&format!("{prefix}.fc2"),
)?;
deepstack_merger_list.push(merger_i);
}
Ok(Self {
spatial_merge_size,
patch_embed,
pos_embed,
num_grid_per_side: num_grid_per_side as u32,
rotary_pos_emb,
blocks,
merger,
deepstack_visual_indexes,
deepstack_merger_list,
dtype: DType::F32,
})
}
pub fn fast_pos_embed_interpolate(&self, grid_thw: &Tensor) -> Result<Tensor> {
let mut idx_list = vec![vec![]; 4];
let mut weight_list = vec![vec![]; 4];
src/models/qwen3vl/processor.rs
-1
View File
@@ -101,7 +101,6 @@ impl Qwen3VLProcessor {
max_frames: 768,
})
}
pub fn new_qwen3_5_default(device: &Device, dtype: DType) -> Result<Self> {
let img_process_cfg = PreprocessorConfig::qwen3_5_img_default();
let video_process_cfg = PreprocessorConfig::qwen3_5_video_default();