tckm.yan/aha
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add qwen3vl 4b, 8b, 32b

Browse Source
This commit is contained in:
Ziyi
2025-11-12 03:20:55 -10:00
parent d15cd45315
commit b5cfcb4684
8 changed files with 49 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -25,7 +25,7 @@
* ubuntu/WSL
```bash
sudo apt-get update
sudo apt-get install -y pkg-config ffmpeg libavutil-dev libavcodec-dev libavformat-dev libavfilter-dev libavdevice-dev libswresample-dev libswscale-dev
sudo apt-get install -y clang pkg-config ffmpeg libavutil-dev libavcodec-dev libavformat-dev libavfilter-dev libavdevice-dev libswresample-dev libswscale-dev
```
* windows参考： https://github.com/zmwangx/rust-ffmpeg/wiki/Notes-on-building
src/main.rs
+6 -2
View File
@@ -77,8 +77,12 @@ async fn main() -> anyhow::Result<()> {
let args = Args::parse();
let model_id = match args.model {
WhichModel::MiniCPM4_0_5B => "OpenBMB/MiniCPM4-0.5B",
WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct",
WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct",
WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct",
WhichModel::Qwen2_5vl7B => "Qwen/Qwen2.5-VL-7B-Instruct",
WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct",
WhichModel::Qwen3vl4B => "Qwen/Qwen3-VL-4B-Instruct",
WhichModel::Qwen3vl8B => "Qwen/Qwen3-VL-8B-Instruct",
WhichModel::Qwen3vl32B => "Qwen/Qwen3-VL-32B-Instruct"
};
let model_path = match args.weight_path {
Some(path) => path,
src/models/mod.rs
+24
View File
@@ -21,8 +21,16 @@ pub enum WhichModel {
MiniCPM4_0_5B,
#[value(name = "qwen2.5vl-3b")]
Qwen2_5vl3B,
#[value(name = "qwen2.5vl-7b")]
Qwen2_5vl7B,
#[value(name = "qwen3vl-2b")]
Qwen3vl2B,
#[value(name = "qwen3vl-4b")]
Qwen3vl4B,
#[value(name = "qwen3vl-8b")]
Qwen3vl8B,
#[value(name = "qwen3vl-32b")]
Qwen3vl32B,
}
pub trait GenerateModel {
@@ -84,9 +92,25 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_
let model = Qwen2_5VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen2_5VL(model)
}
WhichModel::Qwen2_5vl7B => {
let model = Qwen2_5VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen2_5VL(model)
}
WhichModel::Qwen3vl2B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(model)
}
WhichModel::Qwen3vl4B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(model)
}
WhichModel::Qwen3vl8B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(model)
}
WhichModel::Qwen3vl32B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(model)
}
};
Ok(model)
src/models/qwen3vl/config.rs
-1
View File
@@ -42,7 +42,6 @@ pub struct Qwen3VLTextConfig {
pub rms_norm_eps: f64,
pub rope_scaling: RopeScaling,
pub rope_theta: f32,
pub tie_word_embeddings: bool,
pub use_cache: bool,
pub vocab_size: usize,
}
src/models/qwen3vl/generate.rs
-1
View File
@@ -47,7 +47,6 @@ impl<'a> Qwen3VLGenerateModel<'a> {
let pre_processor = Qwen3VLProcessor::new(path, &device, dtype)?;
let model_list = find_type_files(path, "safetensors")?;
let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, &device)? };
let vb = vb.pp("model");
let qwen3_vl = Qwen3VLModel::new(cfg, vb)?;
let generation_config_path = path.to_string() + "/generation_config.json";
let generation_config: Qwen3VLGenerationConfig =
src/models/qwen3vl/model.rs
+8 -7
View File
@@ -568,7 +568,7 @@ impl Qwen3VLTextAttention {
pub fn new(config: Qwen3VLTextConfig, vb: VarBuilder) -> Result<Self> {
let hidden_size = config.hidden_size;
let num_attention_heads = config.num_attention_heads;
let head_dim = hidden_size / num_attention_heads;
let head_dim = config.head_dim;
let num_key_value_heads = config.num_key_value_heads;
let num_kv_groups = num_attention_heads / num_key_value_heads;
let scaling = 1f64 / f64::sqrt(head_dim as f64);
@@ -576,7 +576,7 @@ impl Qwen3VLTextAttention {
let q_proj = linear(hidden_size, num_attention_heads * head_dim, vb.pp("q_proj"))?;
let k_proj = linear(hidden_size, num_key_value_heads * head_dim, vb.pp("k_proj"))?;
let v_proj = linear(hidden_size, num_key_value_heads * head_dim, vb.pp("v_proj"))?;
let o_proj = linear(hidden_size, hidden_size, vb.pp("o_proj"))?;
let o_proj = linear(num_attention_heads * head_dim, hidden_size, vb.pp("o_proj"))?;
(q_proj, k_proj, v_proj, o_proj)
} else {
let q_proj =
@@ -585,7 +585,7 @@ impl Qwen3VLTextAttention {
linear_no_bias(hidden_size, num_key_value_heads * head_dim, vb.pp("k_proj"))?;
let v_proj =
linear_no_bias(hidden_size, num_key_value_heads * head_dim, vb.pp("v_proj"))?;
let o_proj = linear_no_bias(hidden_size, hidden_size, vb.pp("o_proj"))?;
let o_proj = linear_no_bias(num_attention_heads * head_dim, hidden_size, vb.pp("o_proj"))?;
(q_proj, k_proj, v_proj, o_proj)
};
let q_norm = rms_norm(head_dim, config.rms_norm_eps, vb.pp("q_norm"))?;
@@ -652,7 +652,7 @@ impl Qwen3VLTextAttention {
attention_mask,
self.scaling,
)?;
let attn_output = attn_output.reshape((b_sz, q_len, self.hidden_size))?;
let attn_output = attn_output.reshape((b_sz, q_len, self.num_attention_heads*self.head_dim))?;
let attn_output = attn_output.apply(&self.o_proj)?;
Ok(attn_output)
}
@@ -738,7 +738,7 @@ impl Qwen3VLTextModel {
layers.push(layer)
}
let norm = rms_norm(config.hidden_size, config.rms_norm_eps, vb.pp("norm"))?;
let head_dim = config.hidden_size / config.num_attention_heads;
let head_dim = config.head_dim;
let rotary_emb = Qwen3VLTextRotaryEmbedding::new(head_dim, config.rope_theta);
let mrope_section = config.rope_scaling.mrope_section.clone();
Ok(Self {
@@ -823,10 +823,11 @@ pub struct Qwen3VLModel {
impl Qwen3VLModel {
pub fn new(config: Qwen3VLConfig, vb: VarBuilder) -> Result<Self> {
let vb_m = vb.pp("model");
let config = config.clone();
let visual = Qwen3VLVisionModel::new(config.vision_config.clone(), vb.pp("visual"))?;
let visual = Qwen3VLVisionModel::new(config.vision_config.clone(), vb_m.pp("visual"))?;
let language_model =
Qwen3VLTextModel::new(config.text_config.clone(), vb.pp("language_model"))?;
Qwen3VLTextModel::new(config.text_config.clone(), vb_m.pp("language_model"))?;
let lm_head = if config.tie_word_embeddings {
Linear::new(language_model.embed_tokens.embeddings().clone(), None)
} else {
src/utils/mod.rs
+1 -1
View File
@@ -21,7 +21,7 @@ pub fn get_device(device: Option<&Device>) -> Device {
None => {
#[cfg(feature = "cuda")]
{
Device::new_cuda(0).unwrap_or(Device::Cpu)
Device::new_cuda(6).unwrap_or(Device::Cpu)
}
#[cfg(not(feature = "cuda"))]
{
src/utils/tensor_utils.rs
+9 -4
View File
@@ -8,10 +8,15 @@ pub fn prepare_causal_attention_mask(
device: &Device,
) -> Result<Tensor> {
// Sliding window mask?
let mask: Vec<_> = (0..tgt_len)
.flat_map(|i| (0..tgt_len).map(move |j| if i < j { f32::NEG_INFINITY } else { 0. }))
.collect();
let mask = Tensor::from_slice(&mask, (tgt_len, tgt_len), device)?;
// let mask: Vec<f32> = (0..tgt_len)
// .flat_map(|i| (0..tgt_len).map(move |j| if i < j { f32::NEG_INFINITY } else { 0. }))
// .collect();
// let mask = Tensor::from_vec(mask, (tgt_len, tgt_len), device)?;
let arange = Tensor::arange(0u32, tgt_len as u32, device)?;
let arange = arange.unsqueeze(1)?.broadcast_as((tgt_len, tgt_len))?;
let upper_triangle = arange.t()?.gt(&arange)?;
let mask = upper_triangle.where_cond(&Tensor::new(f32::NEG_INFINITY, device)?.broadcast_as(arange.shape())?, &Tensor::new(0f32, device)?.broadcast_as(arange.shape())?)?;
let mask = if seqlen_offset > 0 {
let mask0 = Tensor::zeros((tgt_len, seqlen_offset), DType::F32, device)?;
Tensor::cat(&[&mask0, &mask], D::Minus1)?