diff --git a/README.md b/README.md index aad630b..1dfab41 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ * ubuntu/WSL ```bash sudo apt-get update -sudo apt-get install -y pkg-config ffmpeg libavutil-dev libavcodec-dev libavformat-dev libavfilter-dev libavdevice-dev libswresample-dev libswscale-dev +sudo apt-get install -y clang pkg-config ffmpeg libavutil-dev libavcodec-dev libavformat-dev libavfilter-dev libavdevice-dev libswresample-dev libswscale-dev ``` * windows参考： https://github.com/zmwangx/rust-ffmpeg/wiki/Notes-on-building diff --git a/src/main.rs b/src/main.rs index 7e66122..f25affe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,8 +77,12 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); let model_id = match args.model { WhichModel::MiniCPM4_0_5B => "OpenBMB/MiniCPM4-0.5B", - WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct", - WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct", + WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct", + WhichModel::Qwen2_5vl7B => "Qwen/Qwen2.5-VL-7B-Instruct", + WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct", + WhichModel::Qwen3vl4B => "Qwen/Qwen3-VL-4B-Instruct", + WhichModel::Qwen3vl8B => "Qwen/Qwen3-VL-8B-Instruct", + WhichModel::Qwen3vl32B => "Qwen/Qwen3-VL-32B-Instruct" }; let model_path = match args.weight_path { Some(path) => path, diff --git a/src/models/mod.rs b/src/models/mod.rs index 4bdc359..8f7b3c6 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -21,8 +21,16 @@ pub enum WhichModel { MiniCPM4_0_5B, #[value(name = "qwen2.5vl-3b")] Qwen2_5vl3B, + #[value(name = "qwen2.5vl-7b")] + Qwen2_5vl7B, #[value(name = "qwen3vl-2b")] Qwen3vl2B, + #[value(name = "qwen3vl-4b")] + Qwen3vl4B, + #[value(name = "qwen3vl-8b")] + Qwen3vl8B, + #[value(name = "qwen3vl-32b")] + Qwen3vl32B, } pub trait GenerateModel { @@ -84,9 +92,25 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result { + let model = Qwen2_5VLGenerateModel::init(path, None, None)?; + ModelInstance::Qwen2_5VL(model) + } WhichModel::Qwen3vl2B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(model) + } + WhichModel::Qwen3vl4B => { + let model = Qwen3VLGenerateModel::init(path, None, None)?; + ModelInstance::Qwen3VL(model) + } + WhichModel::Qwen3vl8B => { + let model = Qwen3VLGenerateModel::init(path, None, None)?; + ModelInstance::Qwen3VL(model) + } + WhichModel::Qwen3vl32B => { + let model = Qwen3VLGenerateModel::init(path, None, None)?; + ModelInstance::Qwen3VL(model) } }; Ok(model) diff --git a/src/models/qwen3vl/config.rs b/src/models/qwen3vl/config.rs index 3124181..ee1a1e4 100644 --- a/src/models/qwen3vl/config.rs +++ b/src/models/qwen3vl/config.rs @@ -42,7 +42,6 @@ pub struct Qwen3VLTextConfig { pub rms_norm_eps: f64, pub rope_scaling: RopeScaling, pub rope_theta: f32, - pub tie_word_embeddings: bool, pub use_cache: bool, pub vocab_size: usize, } diff --git a/src/models/qwen3vl/generate.rs b/src/models/qwen3vl/generate.rs index 632e471..e8669b3 100644 --- a/src/models/qwen3vl/generate.rs +++ b/src/models/qwen3vl/generate.rs @@ -47,7 +47,6 @@ impl<'a> Qwen3VLGenerateModel<'a> { let pre_processor = Qwen3VLProcessor::new(path, &device, dtype)?; let model_list = find_type_files(path, "safetensors")?; let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, &device)? }; - let vb = vb.pp("model"); let qwen3_vl = Qwen3VLModel::new(cfg, vb)?; let generation_config_path = path.to_string() + "/generation_config.json"; let generation_config: Qwen3VLGenerationConfig = diff --git a/src/models/qwen3vl/model.rs b/src/models/qwen3vl/model.rs index e556fb6..40e906a 100644 --- a/src/models/qwen3vl/model.rs +++ b/src/models/qwen3vl/model.rs @@ -568,7 +568,7 @@ impl Qwen3VLTextAttention { pub fn new(config: Qwen3VLTextConfig, vb: VarBuilder) -> Result { let hidden_size = config.hidden_size; let num_attention_heads = config.num_attention_heads; - let head_dim = hidden_size / num_attention_heads; + let head_dim = config.head_dim; let num_key_value_heads = config.num_key_value_heads; let num_kv_groups = num_attention_heads / num_key_value_heads; let scaling = 1f64 / f64::sqrt(head_dim as f64); @@ -576,7 +576,7 @@ impl Qwen3VLTextAttention { let q_proj = linear(hidden_size, num_attention_heads * head_dim, vb.pp("q_proj"))?; let k_proj = linear(hidden_size, num_key_value_heads * head_dim, vb.pp("k_proj"))?; let v_proj = linear(hidden_size, num_key_value_heads * head_dim, vb.pp("v_proj"))?; - let o_proj = linear(hidden_size, hidden_size, vb.pp("o_proj"))?; + let o_proj = linear(num_attention_heads * head_dim, hidden_size, vb.pp("o_proj"))?; (q_proj, k_proj, v_proj, o_proj) } else { let q_proj = @@ -585,7 +585,7 @@ impl Qwen3VLTextAttention { linear_no_bias(hidden_size, num_key_value_heads * head_dim, vb.pp("k_proj"))?; let v_proj = linear_no_bias(hidden_size, num_key_value_heads * head_dim, vb.pp("v_proj"))?; - let o_proj = linear_no_bias(hidden_size, hidden_size, vb.pp("o_proj"))?; + let o_proj = linear_no_bias(num_attention_heads * head_dim, hidden_size, vb.pp("o_proj"))?; (q_proj, k_proj, v_proj, o_proj) }; let q_norm = rms_norm(head_dim, config.rms_norm_eps, vb.pp("q_norm"))?; @@ -652,7 +652,7 @@ impl Qwen3VLTextAttention { attention_mask, self.scaling, )?; - let attn_output = attn_output.reshape((b_sz, q_len, self.hidden_size))?; + let attn_output = attn_output.reshape((b_sz, q_len, self.num_attention_heads*self.head_dim))?; let attn_output = attn_output.apply(&self.o_proj)?; Ok(attn_output) } @@ -738,7 +738,7 @@ impl Qwen3VLTextModel { layers.push(layer) } let norm = rms_norm(config.hidden_size, config.rms_norm_eps, vb.pp("norm"))?; - let head_dim = config.hidden_size / config.num_attention_heads; + let head_dim = config.head_dim; let rotary_emb = Qwen3VLTextRotaryEmbedding::new(head_dim, config.rope_theta); let mrope_section = config.rope_scaling.mrope_section.clone(); Ok(Self { @@ -823,10 +823,11 @@ pub struct Qwen3VLModel { impl Qwen3VLModel { pub fn new(config: Qwen3VLConfig, vb: VarBuilder) -> Result { + let vb_m = vb.pp("model"); let config = config.clone(); - let visual = Qwen3VLVisionModel::new(config.vision_config.clone(), vb.pp("visual"))?; + let visual = Qwen3VLVisionModel::new(config.vision_config.clone(), vb_m.pp("visual"))?; let language_model = - Qwen3VLTextModel::new(config.text_config.clone(), vb.pp("language_model"))?; + Qwen3VLTextModel::new(config.text_config.clone(), vb_m.pp("language_model"))?; let lm_head = if config.tie_word_embeddings { Linear::new(language_model.embed_tokens.embeddings().clone(), None) } else { diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 550e54e..960ef26 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -21,7 +21,7 @@ pub fn get_device(device: Option<&Device>) -> Device { None => { #[cfg(feature = "cuda")] { - Device::new_cuda(0).unwrap_or(Device::Cpu) + Device::new_cuda(6).unwrap_or(Device::Cpu) } #[cfg(not(feature = "cuda"))] { diff --git a/src/utils/tensor_utils.rs b/src/utils/tensor_utils.rs index 2da0720..41ffa50 100644 --- a/src/utils/tensor_utils.rs +++ b/src/utils/tensor_utils.rs @@ -8,10 +8,15 @@ pub fn prepare_causal_attention_mask( device: &Device, ) -> Result { // Sliding window mask? - let mask: Vec<_> = (0..tgt_len) - .flat_map(|i| (0..tgt_len).map(move |j| if i < j { f32::NEG_INFINITY } else { 0. })) - .collect(); - let mask = Tensor::from_slice(&mask, (tgt_len, tgt_len), device)?; + // let mask: Vec = (0..tgt_len) + // .flat_map(|i| (0..tgt_len).map(move |j| if i < j { f32::NEG_INFINITY } else { 0. })) + // .collect(); + // let mask = Tensor::from_vec(mask, (tgt_len, tgt_len), device)?; + let arange = Tensor::arange(0u32, tgt_len as u32, device)?; + let arange = arange.unsqueeze(1)?.broadcast_as((tgt_len, tgt_len))?; + let upper_triangle = arange.t()?.gt(&arange)?; + let mask = upper_triangle.where_cond(&Tensor::new(f32::NEG_INFINITY, device)?.broadcast_as(arange.shape())?, &Tensor::new(0f32, device)?.broadcast_as(arange.shape())?)?; + let mask = if seqlen_offset > 0 { let mask0 = Tensor::zeros((tgt_len, seqlen_offset), DType::F32, device)?; Tensor::cat(&[&mask0, &mask], D::Minus1)?