tckm.yan/aha
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unify cargo version and add some ci rules

Browse Source
This commit is contained in:
Yijun Zhao
2025-10-15 21:03:49 +08:00
parent 0fd3c7d935
commit 9b9a8f2c73
40 changed files with 873 additions and 836 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/qwen2_5vl/config.rs
+4 -4
View File
@@ -72,8 +72,8 @@ pub struct VisionSetting {
pub image_std: Vec<f32>,
}
impl VisionSetting {
pub fn default() -> Self {
impl Default for VisionSetting {
fn default() -> Self {
Self {
image_factor: 28,
min_pixels: 4 * 28 * 28,
@@ -90,8 +90,8 @@ impl VisionSetting {
fps: 2.0,
fps_min_frames: 4,
fps_max_frames: 768,
image_mean: vec![0.48145466_f32, 0.4578275, 0.40821073],
image_std: vec![0.26862954, 0.26130258, 0.27577711],
image_mean: vec![0.48145466_f32, 0.4578275f32, 0.40821073f32],
image_std: vec![0.26862954f32, 0.2613026f32, 0.2757771f32],
}
}
}
src/models/qwen2_5vl/generate.rs
+18 -23
View File
@@ -1,16 +1,4 @@
// use crate::models::GenerateStream;
use crate::models::qwen2_5vl::config::Qwen2_5VLConfig;
use crate::utils::utils::{
build_completion_chunk_response, build_completion_response, find_type_files, get_device, get_dtype, get_logit_processor
};
use crate::{
chat_template::chat_template::ChatTemplate,
models::{
GenerateModel,
qwen2_5vl::{model::Qwen2_5VLModel, processor::Qwen2_5VLProcessor},
},
tokenizer::tokenizer::TokenizerModel,
};
use anyhow::{Result, anyhow};
use candle_core::{D, DType, Device, IndexOp, Tensor};
use candle_nn::VarBuilder;
@@ -20,6 +8,20 @@ use openai_dive::v1::resources::chat::{
use rocket::async_stream::stream;
use rocket::futures::Stream;
use crate::models::qwen2_5vl::config::Qwen2_5VLConfig;
use crate::utils::{
build_completion_chunk_response, build_completion_response, find_type_files, get_device,
get_dtype, get_logit_processor,
};
use crate::{
chat_template::ChatTemplate,
models::{
GenerateModel,
qwen2_5vl::{model::Qwen2_5VLModel, processor::Qwen2_5VLProcessor},
},
tokenizer::TokenizerModel,
};
pub struct Qwen2_5VLGenerateModel<'a> {
chat_template: ChatTemplate<'a>,
tokenizer: TokenizerModel,
@@ -43,7 +45,7 @@ impl<'a> Qwen2_5VLGenerateModel<'a> {
let endoftext_id = cfg.bos_token_id;
let im_end_id = cfg.eos_token_id;
// let model_list = find_safetensors_files(&path)?;
let model_list = find_type_files(&path, "safetensors")?;
let model_list = find_type_files(path, "safetensors")?;
let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, device)? };
let qwen2_5_vl = Qwen2_5VLModel::new(cfg, vb)?;
@@ -60,7 +62,6 @@ impl<'a> Qwen2_5VLGenerateModel<'a> {
}
impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p);
let mes_render = self.chat_template.apply_chat_template(&mes)?;
@@ -84,10 +85,7 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
.broadcast_sub(&Tensor::new(vec![1_u32], input_ids.device())?)?;
let mut generate = Vec::new();
let sample_len = match mes.max_tokens {
Some(max) => max,
None => 1024,
};
let sample_len = mes.max_tokens.unwrap_or(1024);
for _ in 0..sample_len {
let logits = self.qwen2_5_vl.forward(
&input_ids,
@@ -145,10 +143,7 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
.to_dtype(candle_core::DType::U32)?
.broadcast_sub(&Tensor::new(vec![1_u32], input_ids.device())?)?;
let sample_len = match mes.max_tokens {
Some(max) => max,
None => 512,
};
let sample_len = mes.max_tokens.unwrap_or(512);
let stream = stream! {
let mut error_tokens = Vec::new();
let mut pixel_values = pixel_values.as_ref();
@@ -170,7 +165,7 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
let next_token = logit_processor.sample(&logits)?;
let mut decode_ids = Vec::new();
if error_tokens.len() > 0 {
if !error_tokens.is_empty() {
decode_ids.extend_from_slice(&error_tokens);
}
decode_ids.push(next_token);
src/models/qwen2_5vl/model.rs
+88 -96
View File
@@ -1,18 +1,21 @@
use crate::{
models::qwen2_5vl::config::{Qwen2_5VLConfig, RopeScaling},
position_embed::rope::{
apply_rotary_pos_emb, apply_rotary_pos_emb_vision, Qwen2_5VLTextRotaryEmbedding, Qwen2_5VisionRotaryEmbedding
},
utils::tensor_utils::{
get_equal_mask, get_vision_next_indices, masked_scatter_dim0, nonzero_index, repeat_kv, safe_arg_sort_last_dim, zero_index
},
};
use anyhow::{Result, anyhow};
use candle_core::{D, DType, Device, IndexOp, Tensor};
use candle_nn::{
Activation, Init, Linear, Module, RmsNorm, VarBuilder, linear, linear_no_bias, rms_norm,
};
use crate::{
models::qwen2_5vl::config::{Qwen2_5VLConfig, RopeScaling},
position_embed::rope::{
Qwen2_5VLTextRotaryEmbedding, Qwen2_5VisionRotaryEmbedding, apply_rotary_pos_emb,
apply_rotary_pos_emb_vision,
},
utils::tensor_utils::{
get_equal_mask, get_vision_next_indices, masked_scatter_dim0, nonzero_index, repeat_kv,
safe_arg_sort_last_dim, zero_index,
},
};
pub struct Qwen2_5VisionPatchEmbed {
conv3d_weight: Tensor,
}
@@ -175,7 +178,7 @@ impl Qwen2_5VLVisionAttention {
let attn_weights = query_states
.matmul(&key_states.transpose(D::Minus2, D::Minus1)?)?
.broadcast_mul(&self.scale)?;
let attn_weights = attn_weights.broadcast_add(&attention_mask)?;
let attn_weights = attn_weights.broadcast_add(attention_mask)?;
let attn_weights = candle_nn::ops::softmax_last_dim(&attn_weights)?;
attn_weights.matmul(&value_states)?
};
@@ -495,7 +498,7 @@ impl Qwen2_5VLVisionModel {
2 => {
let mut cu_seqlens_repeat = Vec::new();
for (index, t) in grid_t.iter().enumerate() {
cu_seqlens_repeat.push(cu_seqlens.i(index)?.repeat(t.clone() as usize)?);
cu_seqlens_repeat.push(cu_seqlens.i(index)?.repeat(*t as usize)?);
}
Tensor::cat(&cu_seqlens_repeat, 0)?.flatten_all()?
}
@@ -521,7 +524,7 @@ impl Qwen2_5VLVisionModel {
hidden_states.device(),
hidden_states.dtype(),
)?;
let mut attention_mask = attention_mask_window.clone();
let mut attention_mask;
for (layer_num, block) in self.blocks.iter().enumerate() {
if self.fullatt_block_indexes.contains(&layer_num) {
attention_mask = attention_mask_full.clone();
@@ -537,7 +540,6 @@ impl Qwen2_5VLVisionModel {
}
}
#[derive(Debug, Clone)]
struct Qwen2_5VLTextMLP {
gate_proj: Linear,
@@ -658,8 +660,7 @@ impl Qwen2_5VLTextAttention {
Some(mask) => attn_weights.broadcast_add(mask)?,
};
let attn_weights = candle_nn::ops::softmax_last_dim(&attn_weights)?;
let attn_weights = attn_weights.matmul(&value_states)?;
attn_weights
attn_weights.matmul(&value_states)?
}
#[cfg(feature = "flash-attn")]
{
@@ -897,12 +898,9 @@ impl Qwen2_5VLModel {
let mut mrope_position_deltas: Vec<i64> = Vec::new();
if image_grid_thw.is_some() || video_grid_thw.is_some() {
let total_input_ids = input_ids.clone();
let mut mask_;
if mask.is_none() {
mask_ = Tensor::ones_like(&total_input_ids)?;
} else {
mask_ = mask.unwrap().clone();
}
let mask_ = mask
.cloned()
.unwrap_or(Tensor::ones_like(&total_input_ids)?);
let mut position_ids = Tensor::ones(
(3, input_ids.dim(0)?, input_ids.dim(1)?),
input_ids.dtype(),
@@ -950,7 +948,7 @@ impl Qwen2_5VLModel {
let llm_grid_h = thw[1] / spatial_merge_size as u32;
let llm_grid_w = thw[2] / spatial_merge_size as u32;
let text_len = text_end - text_start;
let start_idx = if llm_pos_ids_list.len() > 0 {
let start_idx = if !llm_pos_ids_list.is_empty() {
llm_pos_ids_list[llm_pos_ids_list.len() - 1]
.max_all()?
.to_scalar::<u32>()?
@@ -1024,7 +1022,7 @@ impl Qwen2_5VLModel {
};
if text_start < input_ids_i.dim(0)? as u32 {
let start_idx = if llm_pos_ids_list.len() > 0 {
let start_idx = if !llm_pos_ids_list.is_empty() {
llm_pos_ids_list[llm_pos_ids_list.len() - 1]
.max_all()?
.to_scalar::<u32>()?
@@ -1051,66 +1049,61 @@ impl Qwen2_5VLModel {
if mrope_position_deltas.rank() == 1 {
mrope_position_deltas = mrope_position_deltas.unsqueeze(0)?;
}
return Ok((position_ids.contiguous()?, mrope_position_deltas));
} else {
if mask.is_some() {
let mut position_ids = mask
.unwrap()
.to_dtype(candle_core::DType::F64)?
.cumsum(D::Minus1)?
.to_dtype(candle_core::DType::U32)?
.broadcast_sub(&Tensor::new(vec![1_u32], input_ids.device())?)?;
for i in 0..position_ids.dim(0)? {
let mut position_ids_i = position_ids.i(i)?;
let mask_i = mask.unwrap().i(i)?;
// 如果有pad, 将填充位置置为1
// 当bs>1, 可能存在不同序列长度，需要添加pad使seq_len长度一致
if mask_i.sum_all()?.to_scalar::<u32>()? != mask_i.dim(0)? as u32 {
let zero_indices = zero_index(&mask_i)?;
let replace_1 = Tensor::ones(
zero_indices.dim(0)?,
candle_core::DType::U32,
input_ids.device(),
)?;
position_ids_i = position_ids_i
.scatter(&zero_indices, &replace_1, 0)?
.unsqueeze(0)?;
position_ids = position_ids.slice_assign(
&[(i..i + 1), (0..position_ids.dim(1)?)],
&position_ids_i,
)?;
}
}
position_ids = position_ids
.unsqueeze(0)?
.broadcast_as((3, input_ids.dim(0)?, input_ids.dim(1)?))?
.contiguous()?;
let mut mrope_position_deltas = position_ids
.max(0)?
.max(D::Minus1)?
.broadcast_sub(&Tensor::new(
vec![mask.unwrap().dim(D::Minus1)? as u32 - 1],
Ok((position_ids.contiguous()?, mrope_position_deltas))
} else if let Some(mask) = mask {
let mut position_ids = mask
.to_dtype(candle_core::DType::F64)?
.cumsum(D::Minus1)?
.to_dtype(candle_core::DType::U32)?
.broadcast_sub(&Tensor::new(vec![1_u32], input_ids.device())?)?;
for i in 0..position_ids.dim(0)? {
let mut position_ids_i = position_ids.i(i)?;
let mask_i = mask.i(i)?;
// 如果有pad, 将填充位置置为1
// 当bs>1, 可能存在不同序列长度，需要添加pad使seq_len长度一致
if mask_i.sum_all()?.to_scalar::<u32>()? != mask_i.dim(0)? as u32 {
let zero_indices = zero_index(&mask_i)?;
let replace_1 = Tensor::ones(
zero_indices.dim(0)?,
candle_core::DType::U32,
input_ids.device(),
)?)?
.contiguous()?;
if mrope_position_deltas.rank() == 1 {
mrope_position_deltas = mrope_position_deltas.unsqueeze(0)?;
)?;
position_ids_i = position_ids_i
.scatter(&zero_indices, &replace_1, 0)?
.unsqueeze(0)?;
position_ids = position_ids
.slice_assign(&[(i..i + 1), (0..position_ids.dim(1)?)], &position_ids_i)?;
}
return Ok((position_ids, mrope_position_deltas));
} else {
let position_ids =
Tensor::arange(0_u32, input_ids.dim(D::Minus1)? as u32, input_ids.device())?
.unsqueeze(0)?
.unsqueeze(0)?
.broadcast_as((3, input_ids.dim(0)?, input_ids.dim(D::Minus1)?))?
.contiguous()?;
let mrope_position_deltas = Tensor::zeros(
(input_ids.dim(0)?, 1),
input_ids.dtype(),
input_ids.device(),
)?;
Ok((position_ids, mrope_position_deltas))
}
position_ids = position_ids
.unsqueeze(0)?
.broadcast_as((3, input_ids.dim(0)?, input_ids.dim(1)?))?
.contiguous()?;
let mut mrope_position_deltas = position_ids
.max(0)?
.max(D::Minus1)?
.broadcast_sub(&Tensor::new(
vec![mask.dim(D::Minus1)? as u32 - 1],
input_ids.device(),
)?)?
.contiguous()?;
if mrope_position_deltas.rank() == 1 {
mrope_position_deltas = mrope_position_deltas.unsqueeze(0)?;
}
Ok((position_ids, mrope_position_deltas))
} else {
let position_ids =
Tensor::arange(0_u32, input_ids.dim(D::Minus1)? as u32, input_ids.device())?
.unsqueeze(0)?
.unsqueeze(0)?
.broadcast_as((3, input_ids.dim(0)?, input_ids.dim(D::Minus1)?))?
.contiguous()?;
let mrope_position_deltas = Tensor::zeros(
(input_ids.dim(0)?, 1),
input_ids.dtype(),
input_ids.device(),
)?;
Ok((position_ids, mrope_position_deltas))
}
}
@@ -1127,14 +1120,14 @@ impl Qwen2_5VLModel {
second_per_grid_ts: Option<Vec<f32>>,
) -> Result<Tensor> {
// input_ids shape: (bs, seq_len)
let mut inputs_embeds = self.model.embed_tokens.forward(&input_ids)?;
let mut inputs_embeds = self.model.embed_tokens.forward(input_ids)?;
// inputs_embeds shape: (bs, seq_len, hidden_dim)
if pixel_values.is_some() && image_grid_thw.is_some() {
if let Some(pixel_values) = pixel_values
&& let Some(image_grid_thw) = image_grid_thw
{
// image_embed shape: (seq_len, hidden_dim)
let image_embed = self
.visual
.forward(pixel_values.unwrap(), image_grid_thw.unwrap())?;
let vision_mask = get_equal_mask(&input_ids, self.cfg.image_token_id as u32)?;
let image_embed = self.visual.forward(pixel_values, image_grid_thw)?;
let vision_mask = get_equal_mask(input_ids, self.cfg.image_token_id as u32)?;
let n_image_tokens = vision_mask.sum_all()?.to_scalar::<u32>()?;
if n_image_tokens as usize != image_embed.dim(0)? {
@@ -1146,12 +1139,12 @@ impl Qwen2_5VLModel {
}
inputs_embeds = masked_scatter_dim0(&inputs_embeds, &image_embed, &vision_mask)?;
}
if pixel_values_video.is_some() && video_grid_thw.is_some() {
let video_embed = self
.visual
.forward(pixel_values_video.unwrap(), video_grid_thw.unwrap())?;
if let Some(pixel_values_video) = pixel_values_video
&& let Some(video_grid_thw) = video_grid_thw
{
let video_embed = self.visual.forward(pixel_values_video, video_grid_thw)?;
let vision_mask = get_equal_mask(&input_ids, self.cfg.video_token_id as u32)?;
let vision_mask = get_equal_mask(input_ids, self.cfg.video_token_id as u32)?;
let n_video_tokens = vision_mask.sum_all()?.to_scalar::<u32>()?;
if n_video_tokens as usize != video_embed.dim(0)? {
return Err(anyhow!(format!(
@@ -1162,8 +1155,8 @@ impl Qwen2_5VLModel {
}
inputs_embeds = masked_scatter_dim0(&inputs_embeds, &video_embed, &vision_mask)?;
}
let mut position_ids;
let mut rope_deltas;
let position_ids;
let rope_deltas;
if (cache_position.is_some() && cache_position.unwrap().i(0)?.to_scalar::<u32>()? == 0)
|| self.rope_deltas.is_none()
{
@@ -1177,12 +1170,11 @@ impl Qwen2_5VLModel {
self.rope_deltas = Some(rope_deltas);
} else {
let (bs, seq_len, _) = inputs_embeds.dims3()?;
let delta = if cache_position.is_some() {
let delta = if let Some(cache_position) = cache_position {
cache_position
.unwrap()
.i(0)?
.to_dtype(self.rope_deltas.as_ref().unwrap().dtype())?
.broadcast_add(&self.rope_deltas.as_ref().unwrap())?
.broadcast_add(self.rope_deltas.as_ref().unwrap())?
.contiguous()?
.to_dtype(candle_core::DType::U32)?
} else {
src/models/qwen2_5vl/processor.rs
+28 -37
View File
@@ -13,7 +13,7 @@ use crate::{
models::qwen2_5vl::config::VisionSetting,
utils::{
img_utils::get_image,
utils::{ceil_by_factor, floor_by_factor, round_by_factor},
{ceil_by_factor, floor_by_factor, round_by_factor},
},
};
@@ -63,22 +63,15 @@ impl Qwen2_5VLProcessor {
vision_map.insert("image".to_string(), Vec::new());
vision_map.insert("video".to_string(), Vec::new());
for chat_mes in mes.messages.clone() {
match chat_mes {
ChatMessage::User { content, name } => match content {
ChatMessageContent::ContentPart(part_vec) => {
for part in part_vec {
match part {
ChatMessageContentPart::Image(img_part) => {
let img_url = img_part.image_url;
vision_map.get_mut("image").unwrap().push(img_url.url);
}
_ => {}
}
}
if let ChatMessage::User { content, .. } = chat_mes
&& let ChatMessageContent::ContentPart(part_vec) = content
{
for part in part_vec {
if let ChatMessageContentPart::Image(img_part) = part {
let img_url = img_part.image_url;
vision_map.get_mut("image").unwrap().push(img_url.url);
}
_ => {}
},
_ => {}
}
}
}
Ok(vision_map)
@@ -107,9 +100,7 @@ impl Qwen2_5VLProcessor {
// 0-255 rescale to 0-1
let img_tensor = img_tensor.affine(1.0 / 255.0, 0.)?;
// normalize
let img_tensor = img_tensor
.broadcast_sub(&img_mean)?
.broadcast_div(&img_std)?;
let img_tensor = img_tensor.broadcast_sub(img_mean)?.broadcast_div(img_std)?;
// (c, h, w) => (1, c, h, w)
let img_tensor = img_tensor.unsqueeze(0)?;
Ok(img_tensor)
@@ -169,7 +160,7 @@ impl Qwen2_5VLProcessor {
let mut vision_grid_thws_vec = Vec::new();
for img in imgs {
let img_tensor = self.process_img(&img, &img_mean, &img_std)?;
let img_tensor = self.process_img(&img, img_mean, img_std)?;
let img_tensor = Tensor::cat(&[&img_tensor, &img_tensor], 0)?.contiguous()?;
let (img_tensor, grid_thw) = self.process_vision_tensor(&img_tensor)?;
pixel_values_vec.push(img_tensor);
@@ -196,8 +187,8 @@ impl Qwen2_5VLProcessor {
let video_tensor = single_video.to_dtype(self.dtype)?.affine(1.0 / 255.0, 0.)?;
// normalize
let video_tensor = video_tensor
.broadcast_sub(&img_mean)?
.broadcast_div(&img_std)?
.broadcast_sub(img_mean)?
.broadcast_div(img_std)?
.contiguous()?;
let (video_tensor, video_grid_thw) = self.process_vision_tensor(&video_tensor)?;
pixel_values_vec.push(video_tensor);
@@ -238,7 +229,7 @@ impl Qwen2_5VLProcessor {
Err(e) => println!("get_image err: {:?}", e),
};
}
if file_vec.len() > 0 {
if !file_vec.is_empty() {
let vision_input = self.process_images(file_vec, &img_mean, &img_std);
match vision_input {
Ok(img_input) => {
@@ -258,7 +249,7 @@ impl Qwen2_5VLProcessor {
Err(e) => println!("get_video_data err: {:?}", e),
};
}
if file_vec.len() > 0 {
if !file_vec.is_empty() {
let vision_input = self.process_videos(file_vec, &img_mean, &img_std);
match vision_input {
Ok(video_input) => {
@@ -280,10 +271,10 @@ impl Qwen2_5VLProcessor {
}
let merge_length = self.vision_setting.merge_size.pow(2);
let mut text = text.to_string();
if image_grid_thw.is_some() {
if let Some(ref image_grid_thw) = image_grid_thw {
let mut index = 0;
while text.contains(&self.image_token) {
let grid_i = image_grid_thw.as_ref().unwrap().i(index)?;
let grid_i = image_grid_thw.i(index)?;
let repeat_num =
grid_i.to_vec1::<u32>()?.iter().product::<u32>() as usize / merge_length;
let replace = "<|placeholder|>".repeat(repeat_num);
@@ -292,10 +283,10 @@ impl Qwen2_5VLProcessor {
}
text = text.replace("<|placeholder|>", &self.image_token);
}
if video_grid_thw.is_some() {
if let Some(ref video_grid_thw) = video_grid_thw {
let mut index = 0;
while text.contains(&self.video_token) {
let grid_i = video_grid_thw.as_ref().unwrap().i(index)?;
let grid_i = video_grid_thw.i(index)?;
let repeat_num =
grid_i.to_vec1::<u32>()?.iter().product::<u32>() as usize / merge_length;
let replace = "<|placeholder|>".repeat(repeat_num);
@@ -336,15 +327,15 @@ pub fn smart_resize(
}
let mut h_bar = std::cmp::max(image_factor, round_by_factor(img_h, image_factor));
let mut w_bar = std::cmp::max(image_factor, round_by_factor(img_w, image_factor));
let mut max_pixels = 0u32;
let mut min_pixels = 0u32;
if is_img {
min_pixels = vision_setting.min_pixels;
max_pixels = vision_setting.max_pixels;
let (min_pixels, max_pixels) = if is_img {
(vision_setting.min_pixels, vision_setting.max_pixels)
} else {
min_pixels = vision_setting.video_min_pixels;
max_pixels = vision_setting.video_max_pixels;
}
(
vision_setting.video_min_pixels,
vision_setting.video_max_pixels,
)
};
if h_bar * w_bar > max_pixels {
let beta = ((img_h * img_w) as f32 / max_pixels as f32).sqrt();
h_bar = floor_by_factor(img_h as f32 / beta, image_factor);
@@ -419,7 +410,7 @@ pub fn get_video_data(
|decoder: &mut ffmpeg::decoder::Video| -> Result<()> {
let mut decoded = ffmpeg::frame::Video::empty();
while decoder.receive_frame(&mut decoded).is_ok() {
if frame_id % sample_interval == 0 {
if frame_id.is_multiple_of(sample_interval) {
let mut rgb_frame = ffmpeg::frame::Video::empty();
scaler
.run(&decoded, &mut rgb_frame)