From 4e7573e4ff26c38b36db68e031ec87c3fc8c4ea0 Mon Sep 17 00:00:00 2001 From: jhqxxx Date: Thu, 13 Nov 2025 01:03:52 +0800 Subject: [PATCH] update readme and fmt --- README.md | 10 +++++++--- src/main.rs | 10 +++++----- src/models/mod.rs | 6 +++--- src/models/qwen3vl/model.rs | 8 ++++---- src/utils/mod.rs | 2 +- src/utils/tensor_utils.rs | 5 ++++- 6 files changed, 24 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 1dfab41..e41e49f 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,13 @@ cargo run -F cuda -- [参数] -m, --model * 指定要加载的模型类型 * 可选值： - * minicpm4-0.5b：MiniCPM4-0.5B 模型 - * qwen2.5vl-3b：Qwen2.5-VL-3B 模型 - * qwen3vl-2b：Qwen3-VL-2B 模型 + * minicpm4-0.5b：OpenBMB/MiniCPM4-0.5B 模型 + * qwen2.5vl-3b：Qwen/Qwen2.5-VL-3B-Instruct 模型 + * qwen2.5vl-7b：Qwen/Qwen2.5-VL-7B-Instruct 模型 + * qwen3vl-2b：Qwen/Qwen3-VL-2B-Instruct 模型 + * qwen3vl-4b：Qwen/Qwen3-VL-4B-Instruct 模型 + * qwen3vl-8b：Qwen/Qwen3-VL-8B-Instruct 模型 + * qwen3vl-32b：Qwen/Qwen3-VL-32B-Instruct 模型 * 示例：--model minicpm4-0.5b 或 -m qwen3vl-2b 3. 权重路径 diff --git a/src/main.rs b/src/main.rs index f25affe..9bd2930 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,12 +77,12 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); let model_id = match args.model { WhichModel::MiniCPM4_0_5B => "OpenBMB/MiniCPM4-0.5B", - WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct", + WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct", WhichModel::Qwen2_5vl7B => "Qwen/Qwen2.5-VL-7B-Instruct", - WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct", - WhichModel::Qwen3vl4B => "Qwen/Qwen3-VL-4B-Instruct", - WhichModel::Qwen3vl8B => "Qwen/Qwen3-VL-8B-Instruct", - WhichModel::Qwen3vl32B => "Qwen/Qwen3-VL-32B-Instruct" + WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct", + WhichModel::Qwen3vl4B => "Qwen/Qwen3-VL-4B-Instruct", + WhichModel::Qwen3vl8B => "Qwen/Qwen3-VL-8B-Instruct", + WhichModel::Qwen3vl32B => "Qwen/Qwen3-VL-32B-Instruct", }; let model_path = match args.weight_path { Some(path) => path, diff --git a/src/models/mod.rs b/src/models/mod.rs index 8f7b3c6..5a5824c 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -99,15 +99,15 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(model) - } + } WhichModel::Qwen3vl4B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(model) - } + } WhichModel::Qwen3vl8B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(model) - } + } WhichModel::Qwen3vl32B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(model) diff --git a/src/models/qwen3vl/model.rs b/src/models/qwen3vl/model.rs index 40e906a..a9736da 100644 --- a/src/models/qwen3vl/model.rs +++ b/src/models/qwen3vl/model.rs @@ -559,7 +559,6 @@ pub struct Qwen3VLTextAttention { num_key_value_heads: usize, num_kv_groups: usize, head_dim: usize, - hidden_size: usize, scaling: f64, kv_cache: Option<(Tensor, Tensor)>, } @@ -585,7 +584,8 @@ impl Qwen3VLTextAttention { linear_no_bias(hidden_size, num_key_value_heads * head_dim, vb.pp("k_proj"))?; let v_proj = linear_no_bias(hidden_size, num_key_value_heads * head_dim, vb.pp("v_proj"))?; - let o_proj = linear_no_bias(num_attention_heads * head_dim, hidden_size, vb.pp("o_proj"))?; + let o_proj = + linear_no_bias(num_attention_heads * head_dim, hidden_size, vb.pp("o_proj"))?; (q_proj, k_proj, v_proj, o_proj) }; let q_norm = rms_norm(head_dim, config.rms_norm_eps, vb.pp("q_norm"))?; @@ -601,7 +601,6 @@ impl Qwen3VLTextAttention { num_key_value_heads, num_kv_groups, head_dim, - hidden_size, scaling, kv_cache: None, }) @@ -652,7 +651,8 @@ impl Qwen3VLTextAttention { attention_mask, self.scaling, )?; - let attn_output = attn_output.reshape((b_sz, q_len, self.num_attention_heads*self.head_dim))?; + let attn_output = + attn_output.reshape((b_sz, q_len, self.num_attention_heads * self.head_dim))?; let attn_output = attn_output.apply(&self.o_proj)?; Ok(attn_output) } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 960ef26..550e54e 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -21,7 +21,7 @@ pub fn get_device(device: Option<&Device>) -> Device { None => { #[cfg(feature = "cuda")] { - Device::new_cuda(6).unwrap_or(Device::Cpu) + Device::new_cuda(0).unwrap_or(Device::Cpu) } #[cfg(not(feature = "cuda"))] { diff --git a/src/utils/tensor_utils.rs b/src/utils/tensor_utils.rs index 41ffa50..cbc80c1 100644 --- a/src/utils/tensor_utils.rs +++ b/src/utils/tensor_utils.rs @@ -15,7 +15,10 @@ pub fn prepare_causal_attention_mask( let arange = Tensor::arange(0u32, tgt_len as u32, device)?; let arange = arange.unsqueeze(1)?.broadcast_as((tgt_len, tgt_len))?; let upper_triangle = arange.t()?.gt(&arange)?; - let mask = upper_triangle.where_cond(&Tensor::new(f32::NEG_INFINITY, device)?.broadcast_as(arange.shape())?, &Tensor::new(0f32, device)?.broadcast_as(arange.shape())?)?; + let mask = upper_triangle.where_cond( + &Tensor::new(f32::NEG_INFINITY, device)?.broadcast_as(arange.shape())?, + &Tensor::new(0f32, device)?.broadcast_as(arange.shape())?, + )?; let mask = if seqlen_offset > 0 { let mask0 = Tensor::zeros((tgt_len, seqlen_offset), DType::F32, device)?;