diff --git a/src/models/common/mod.rs b/src/models/common/mod.rs index 786cf0b..f306893 100644 --- a/src/models/common/mod.rs +++ b/src/models/common/mod.rs @@ -280,17 +280,17 @@ pub fn eager_attention_forward( ) -> Result { let key_states = match num_key_value_groups { Some(g) => repeat_kv(key_states.clone(), g)?.contiguous()?, - None => key_states.clone() + None => key_states.clone(), }; let value_states = match num_key_value_groups { Some(g) => repeat_kv(value_states.clone(), g)?.contiguous()?, - None => value_states.clone() + None => value_states.clone(), }; let attn_output = { #[cfg(not(feature = "flash-attn"))] { let attn_weights = query_states.matmul(&key_states.transpose(D::Minus2, D::Minus1)?)?; - let attn_weights = (attn_weights * scaling)?; + let attn_weights = (attn_weights * scaling)?; let attn_weights = match attention_mask { None => attn_weights, Some(mask) => attn_weights.broadcast_add(&mask.to_dtype(attn_weights.dtype())?)?, @@ -317,6 +317,6 @@ pub fn eager_attention_forward( } }; let attn_output = attn_output.transpose(1, 2)?.contiguous()?; - + Ok(attn_output) } diff --git a/src/models/mod.rs b/src/models/mod.rs index 4ec507b..f0bc8e4 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -1,8 +1,8 @@ pub mod common; pub mod minicpm4; pub mod qwen2_5vl; -pub mod voxcpm; pub mod qwen3vl; +pub mod voxcpm; use anyhow::Result; use openai_dive::v1::resources::chat::{ diff --git a/src/models/qwen2_5vl/generate.rs b/src/models/qwen2_5vl/generate.rs index c189258..0bbdde3 100644 --- a/src/models/qwen2_5vl/generate.rs +++ b/src/models/qwen2_5vl/generate.rs @@ -118,7 +118,7 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> { let response = build_completion_response(res, "qwen2.5vl"); Ok(response) } - + fn generate_stream( &mut self, mes: ChatCompletionParameters, diff --git a/src/models/qwen3vl/config.rs b/src/models/qwen3vl/config.rs index 57d241f..3124181 100644 --- a/src/models/qwen3vl/config.rs +++ b/src/models/qwen3vl/config.rs @@ -1,6 +1,5 @@ use candle_nn::Activation; - #[derive(Debug, Clone, PartialEq, serde::Deserialize)] pub struct Size { pub longest_edge: usize, @@ -86,4 +85,4 @@ pub struct Qwen3VLGenerationConfig { pub top_k: usize, pub temperature: f32, pub repetition_penalty: f32, -} \ No newline at end of file +} diff --git a/src/models/qwen3vl/generate.rs b/src/models/qwen3vl/generate.rs index 122ec71..ae67c80 100644 --- a/src/models/qwen3vl/generate.rs +++ b/src/models/qwen3vl/generate.rs @@ -1,20 +1,26 @@ use anyhow::{Result, anyhow}; use candle_core::{DType, Device, Tensor}; use candle_nn::VarBuilder; -use openai_dive::v1::resources::chat::{ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse}; -use rocket::futures::Stream; +use openai_dive::v1::resources::chat::{ + ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse, +}; use rocket::async_stream::stream; +use rocket::futures::Stream; use crate::{ chat_template::ChatTemplate, - models::{GenerateModel, qwen3vl::{ - config::{Qwen3VLConfig, Qwen3VLGenerationConfig}, - model::Qwen3VLModel, - processor::Qwen3VLProcessor, - }}, + models::{ + GenerateModel, + qwen3vl::{ + config::{Qwen3VLConfig, Qwen3VLGenerationConfig}, + model::Qwen3VLModel, + processor::Qwen3VLProcessor, + }, + }, tokenizer::TokenizerModel, utils::{ - build_completion_chunk_response, build_completion_response, find_type_files, get_device, get_dtype, get_logit_processor + build_completion_chunk_response, build_completion_response, find_type_files, get_device, + get_dtype, get_logit_processor, }, }; @@ -57,7 +63,6 @@ impl<'a> Qwen3VLGenerateModel<'a> { generation_config, }) } - } impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> { @@ -196,4 +201,4 @@ impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> { }; Ok(stream) } -} \ No newline at end of file +} diff --git a/src/models/qwen3vl/mod.rs b/src/models/qwen3vl/mod.rs index 75b27e4..8b1baf7 100644 --- a/src/models/qwen3vl/mod.rs +++ b/src/models/qwen3vl/mod.rs @@ -1,4 +1,4 @@ -pub mod processor; pub mod config; +pub mod generate; pub mod model; -pub mod generate; \ No newline at end of file +pub mod processor; diff --git a/src/models/qwen3vl/model.rs b/src/models/qwen3vl/model.rs index 0f4eb33..5b2c8f6 100644 --- a/src/models/qwen3vl/model.rs +++ b/src/models/qwen3vl/model.rs @@ -1135,7 +1135,7 @@ impl Qwen3VLModel { let (image_embeds, deepstack_img_embed) = self.get_vision_features(pixel_values, image_grid_thw)?; let image_embeds = Tensor::cat(&image_embeds, 0)?; - let vision_mask = self.get_placeholder_mask(&input_ids, true)?; + let vision_mask = self.get_placeholder_mask(input_ids, true)?; let n_image_tokens = vision_mask.sum_all()?.to_scalar::()?; if n_image_tokens as usize != image_embeds.dim(0)? { return Err(anyhow!(format!( @@ -1154,7 +1154,7 @@ impl Qwen3VLModel { let (video_embeds, deepstack_video_embed) = self.get_vision_features(pixel_values_video, video_grid_thw)?; let video_embeds = Tensor::cat(&video_embeds, 0)?; - let vision_mask = self.get_placeholder_mask(&input_ids, false)?; + let vision_mask = self.get_placeholder_mask(input_ids, false)?; let n_video_tokens = vision_mask.sum_all()?.to_scalar::()?; if n_video_tokens as usize != video_embeds.dim(0)? { return Err(anyhow!(format!( @@ -1190,8 +1190,8 @@ impl Qwen3VLModel { img_embed.dtype(), img_embed.device(), )?; - let embed_joint = embed_joint.index_add(&image_nonzero_joint, &img_embed, 0)?; - let embed_joint = embed_joint.index_add(&video_nonzero_joint, &vid_embed, 0)?; + let embed_joint = embed_joint.index_add(&image_nonzero_joint, img_embed, 0)?; + let embed_joint = embed_joint.index_add(&video_nonzero_joint, vid_embed, 0)?; deepstack_embeds.push(embed_joint); } visual_pos_mask = Some(visual_mask); diff --git a/src/models/qwen3vl/processor.rs b/src/models/qwen3vl/processor.rs index 2e50526..b190626 100644 --- a/src/models/qwen3vl/processor.rs +++ b/src/models/qwen3vl/processor.rs @@ -259,7 +259,7 @@ impl Qwen3VLProcessor { fps: f32, t_merge_size: usize, ) -> Result> { - let indices = if frames_indices.len() % t_merge_size != 0 { + let indices = if !frames_indices.len().is_multiple_of(t_merge_size) { let mut frames_indices = frames_indices.clone(); let last = frames_indices[frames_indices.len() - 1]; let pad_len = t_merge_size - frames_indices.len() % t_merge_size; @@ -385,12 +385,15 @@ impl Qwen3VLProcessor { let frame_seqlen = h * w / merge_length as u32; for frame_idx in 0..t { let curr_time = curr_timestamp[frame_idx as usize]; - video_placeholder = video_placeholder + format!("<{:.1} seconds>", curr_time).as_str(); - video_placeholder = video_placeholder + self.vision_start_token.as_str(); - video_placeholder = video_placeholder + "<|placeholder|>".repeat(frame_seqlen as usize).as_str(); - video_placeholder = video_placeholder + self.vision_end_token.as_str(); + video_placeholder += format!("<{:.1} seconds>", curr_time).as_str(); + video_placeholder += self.vision_start_token.as_str(); + video_placeholder += "<|placeholder|>".repeat(frame_seqlen as usize).as_str(); + video_placeholder += self.vision_end_token.as_str(); } - let three_token = format!("{}{}{}", self.vision_start_token, self.video_token, self.vision_end_token); + let three_token = format!( + "{}{}{}", + self.vision_start_token, self.video_token, self.vision_end_token + ); if text.contains(&three_token) { text = text.replacen(&three_token, &video_placeholder, 1); } else { diff --git a/src/utils/tensor_utils.rs b/src/utils/tensor_utils.rs index 8d9f9ae..2da0720 100644 --- a/src/utils/tensor_utils.rs +++ b/src/utils/tensor_utils.rs @@ -247,7 +247,7 @@ pub fn bitor_tensor(mask1: &Tensor, mask2: &Tensor) -> Result { mask1.shape() == mask2.shape(), " bitor_tensor two tensor shape mask be equal" ); - let bitor = mask1.add(&mask2)?.ne(&Tensor::zeros_like(&mask1)?)?; + let bitor = mask1.add(mask2)?.ne(&Tensor::zeros_like(mask1)?)?; Ok(bitor) } @@ -256,38 +256,37 @@ pub fn prod_tensor_last_dim(t: &Tensor) -> Result { 0 => t.clone(), 1 => { let data_type = t.dtype(); - let t_prod = match data_type { + match data_type { DType::U8 => { let t_vec = t.to_vec1::()?; let prod = t_vec.iter().product::(); - Tensor::from_slice(&vec![prod], 1, t.device())? + Tensor::from_slice(&[prod], 1, t.device())? } DType::U32 => { let t_vec = t.to_vec1::()?; let prod = t_vec.iter().product::(); - Tensor::from_slice(&vec![prod], 1, t.device())? + Tensor::from_slice(&[prod], 1, t.device())? } DType::I64 => { let t_vec = t.to_vec1::()?; let prod = t_vec.iter().product::(); - Tensor::from_slice(&vec![prod], 1, t.device())? + Tensor::from_slice(&[prod], 1, t.device())? } DType::F64 => { let t_vec = t.to_vec1::()?; let prod = t_vec.iter().product::(); - Tensor::from_slice(&vec![prod], 1, t.device())? + Tensor::from_slice(&[prod], 1, t.device())? } _ => { let t_vec = t.to_vec1::()?; let prod = t_vec.iter().product::(); - Tensor::from_slice(&vec![prod], 1, t.device())? + Tensor::from_slice(&[prod], 1, t.device())? } - }; - t_prod + } } 2 => { let data_type = t.dtype(); - let t_prod = match data_type { + match data_type { DType::U8 => { let t_vec = t.to_vec2::()?; let mut prod_vec = vec![]; @@ -333,8 +332,7 @@ pub fn prod_tensor_last_dim(t: &Tensor) -> Result { } Tensor::new(prod_vec, t.device())? } - }; - t_prod + } } _ => { return Err(anyhow!(format!("can not action this dim"))); @@ -343,12 +341,8 @@ pub fn prod_tensor_last_dim(t: &Tensor) -> Result { Ok(prod) } -pub fn mask_index_add( - original: &Tensor, - mask: &Tensor, - add: &Tensor, -) -> Result { - let visual_nonzero_index = nonzero_index(&mask)?; +pub fn mask_index_add(original: &Tensor, mask: &Tensor, add: &Tensor) -> Result { + let visual_nonzero_index = nonzero_index(mask)?; let xs = original.index_add(&visual_nonzero_index, add, 0)?; Ok(xs) } diff --git a/tests/config_tests.rs b/tests/config_tests.rs index 7893df9..e2b4b66 100644 --- a/tests/config_tests.rs +++ b/tests/config_tests.rs @@ -1,5 +1,6 @@ use aha::models::{ - minicpm4::config::MiniCPM4Config, qwen2_5vl::config::Qwen2_5VLConfig, qwen3vl::config::Qwen3VLConfig, voxcpm::config::VoxCPMConfig + minicpm4::config::MiniCPM4Config, qwen2_5vl::config::Qwen2_5VLConfig, + qwen3vl::config::Qwen3VLConfig, voxcpm::config::VoxCPMConfig, }; use anyhow::Result; diff --git a/tests/messy_test.rs b/tests/messy_test.rs index 96940c8..86fec2c 100644 --- a/tests/messy_test.rs +++ b/tests/messy_test.rs @@ -1,4 +1,4 @@ -use aha::utils::{tensor_utils::bitor_tensor}; +use aha::utils::tensor_utils::bitor_tensor; use anyhow::Result; use candle_core::Tensor; diff --git a/tests/weight_test.rs b/tests/weight_test.rs index d16e461..1ec0928 100644 --- a/tests/weight_test.rs +++ b/tests/weight_test.rs @@ -50,7 +50,7 @@ fn voxcpm_weight() -> Result<()> { fn qwen3vl_weight() -> Result<()> { let model_path = "/home/jhq/huggingface_model/Qwen/Qwen3-VL-4B-Instruct/"; let model_list = find_type_files(model_path, "safetensors")?; - + let device = Device::Cpu; for m in &model_list { let weights = safetensors::load(m, &device)?; @@ -60,4 +60,4 @@ fn qwen3vl_weight() -> Result<()> { } println!("model_list: {:?}", model_list); Ok(()) -} \ No newline at end of file +}