tckm.yan/aha
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update qwen2.5vl

Browse Source
This commit is contained in:
jhqxxx
2025-09-22 23:49:58 +08:00
parent 0b0076cdc9
commit fdfdfedeb4
7 changed files with 1300 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+944 -23
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+6 -1
View File
@@ -21,7 +21,12 @@ base64 = "0.22.1"
num = "0.4.3"
minijinja = "2.12.0"
tokenizers = "0.22.1"
openai_dive = "1.3.0"
openai_dive = { version = "1.3.0", features = ["stream"]}
uuid = { version = "1.18.1", features = ["v4"]}
chrono = "0.4.42"
rocket = "0.5.1"
tokio = "1.47.1"
[features]
flash-attn=["candle-flash-attn"]
cuda=["candle-nn/cuda", "candle-core/cuda", "candle-transformers/cuda"]
src/lib.rs
+1 -1
View File
@@ -17,7 +17,7 @@ impl ModelType {
model_path: &str,
device: Option<&Device>,
dtype: Option<DType>,
) -> Result<Box<dyn GenerateModel>> {
) -> Result<Box<impl GenerateModel>> {
match model_type {
ModelType::Qwen2_5VL => {
let model = Qwen2_5VLGenerateModel::init(model_path, device, dtype)?;
src/models/mod.rs
+5 -2
View File
@@ -1,11 +1,14 @@
pub mod qwen2_5vl;
use anyhow::Result;
use candle_core::{DType, Device};
use openai_dive::v1::resources::chat::ChatCompletionParameters;
use openai_dive::v1::resources::chat::{ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse};
use rocket::futures::Stream;
pub trait GenerateModel {
fn init(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self>
where
Self: Sized;
fn generate(&mut self, mes: ChatCompletionParameters) -> Result<String>;
fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse>;
fn generate_stream(&mut self, mes: ChatCompletionParameters) -> Result<impl Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>>;
}
src/models/qwen2_5vl/generate.rs
+117 -44
View File
@@ -1,5 +1,9 @@
// use crate::models::GenerateStream;
use crate::models::qwen2_5vl::config::Config;
use crate::utils::utils::{find_safetensors_files, get_device, get_dtype};
use crate::utils::utils::{
build_completion_chunk_response, build_completion_response, find_safetensors_files, get_device,
get_dtype, get_logit_processor,
};
use crate::{
chat_template::chat_template::ChatTemplate,
models::{
@@ -12,7 +16,11 @@ use anyhow::{Result, anyhow};
use candle_core::{D, DType, Device, IndexOp, Tensor};
use candle_nn::VarBuilder;
use candle_transformers::generation::LogitsProcessor;
use openai_dive::v1::resources::chat::ChatCompletionParameters;
use openai_dive::v1::resources::chat::{
ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse,
};
use rocket::async_stream::stream;
use rocket::futures::Stream;
pub struct Qwen2_5VLGenerateModel<'a> {
chat_template: ChatTemplate<'a>,
@@ -20,7 +28,8 @@ pub struct Qwen2_5VLGenerateModel<'a> {
pre_processor: Qwen2_5VLProcessor,
qwen2_5_vl: Qwen2_5VLModel,
device: Device,
dtype: DType,
endoftext_id: u32,
im_end_id: u32,
}
impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
@@ -33,29 +42,25 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
let cfg_dtype = cfg.torch_dtype.as_str();
let dtype = get_dtype(dtype, cfg_dtype);
let pre_processor = Qwen2_5VLProcessor::new(device, dtype)?;
let endoftext_id = cfg.bos_token_id as u32;
let im_end_id = cfg.eos_token_id as u32;
let model_list = find_safetensors_files(&path)?;
let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, device)? };
let qwen2_5_vl = Qwen2_5VLModel::new(cfg, vb)?;
Ok(Qwen2_5VLGenerateModel {
chat_template,
tokenizer,
pre_processor,
qwen2_5_vl,
device: device.clone(),
dtype: dtype,
endoftext_id,
im_end_id,
})
}
fn generate(&mut self, mes: ChatCompletionParameters) -> Result<String> {
let temperature = match mes.temperature {
Some(temp) => Some(temp as f64),
None => None,
};
let top_p = match mes.top_p {
Some(tp) => Some(tp as f64),
None => None,
};
let mut logit_processor = LogitsProcessor::new(34562, temperature, top_p);
fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p);
let mes_render = self.chat_template.apply_chat_template(&mes)?;
let input = self.pre_processor.process_info(&mes, &mes_render)?;
let mut input_ids = self
@@ -63,33 +68,11 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
.text_encode(input.replace_text.clone(), &self.device)?;
let mut seq_len = input_ids.dim(1)?;
let mut seqlen_offset = 0;
let end_of_text_id = self.qwen2_5_vl.cfg.bos_token_id as u32;
let im_end_id = self.qwen2_5_vl.cfg.eos_token_id as u32;
let mut pixel_values = if input.pixel_values.is_some() {
Some(&input.pixel_values.unwrap().clone())
} else {
None
};
let image_grid_thw = if input.image_grid_thw.is_some() {
Some(&input.image_grid_thw.unwrap().clone())
} else {
None
};
let mut pixel_values_video = if input.pixel_values_video.is_some() {
Some(&input.pixel_values_video.unwrap().clone())
} else {
None
};
let video_grid_thw = if input.video_grid_thw.is_some() {
Some(&input.video_grid_thw.unwrap().clone())
} else {
None
};
let second_per_grid_ts = if input.second_per_grid_ts.is_some() {
Some(input.second_per_grid_ts.unwrap().clone())
} else {
None
};
let mut pixel_values = input.pixel_values.as_ref();
let image_grid_thw = input.image_grid_thw.as_ref();
let mut pixel_values_video = input.pixel_values_video.as_ref();
let video_grid_thw = input.video_grid_thw.as_ref();
let second_per_grid_ts = input.second_per_grid_ts.clone();
let mut mask = Tensor::ones_like(&input_ids)?;
let mut cache_position = Tensor::ones_like(&input_ids.i(0)?)?
@@ -118,7 +101,7 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
let next_token = logit_processor.sample(&logits)?;
generate.push(next_token);
if next_token == end_of_text_id || next_token == im_end_id {
if next_token == self.endoftext_id || next_token == self.im_end_id {
break;
}
seqlen_offset += seq_len;
@@ -132,6 +115,96 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
}
let res = self.tokenizer.token_decode(generate)?;
self.qwen2_5_vl.clear_kv_cache();
Ok(res)
let response = build_completion_response(res, "qwen2.5vl");
Ok(response)
}
fn generate_stream(
&mut self,
mes: ChatCompletionParameters,
) -> Result<impl Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>> {
let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p);
let mes_render = self.chat_template.apply_chat_template(&mes)?;
let input = self.pre_processor.process_info(&mes, &mes_render)?;
let mut input_ids = self
.tokenizer
.text_encode(input.replace_text.clone(), &self.device)?;
let mut seq_len = input_ids.dim(1)?;
let mut seqlen_offset = 0;
let pixel_values = input.pixel_values.clone();
let image_grid_thw = input.image_grid_thw.clone();
let pixel_values_video = input.pixel_values_video.clone();
let video_grid_thw = input.video_grid_thw.clone();
let second_per_grid_ts = input.second_per_grid_ts.clone();
let mut mask = Tensor::ones_like(&input_ids)?;
let mut cache_position = Tensor::ones_like(&input_ids.i(0)?)?
.to_dtype(candle_core::DType::F64)?
.cumsum(D::Minus1)?
.to_dtype(candle_core::DType::U32)?
.broadcast_sub(&Tensor::new(vec![1_u32], input_ids.device())?)?;
let sample_len = match mes.max_tokens {
Some(max) => max,
None => 512,
};
let stream = stream! {
let mut error_tokens = Vec::new();
let mut pixel_values = pixel_values.as_ref();
let image_grid_thw = image_grid_thw.as_ref();
let mut pixel_values_video = pixel_values_video.as_ref();
let video_grid_thw = video_grid_thw.as_ref();
for _ in 0..sample_len {
let logits = self.qwen2_5_vl.forward(
&input_ids,
pixel_values,
image_grid_thw,
pixel_values_video,
video_grid_thw,
&mask,
Some(&cache_position),
seqlen_offset,
second_per_grid_ts.clone(),
)?;
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
let next_token = logit_processor.sample(&logits)?;
let mut decode_ids = Vec::new();
if error_tokens.len() > 0 {
decode_ids.extend_from_slice(&error_tokens);
}
decode_ids.push(next_token);
let decoded_token = self.tokenizer.token_decode(decode_ids).map_err(|e| anyhow!(format!("stream decode error{}", e)))?;
if decoded_token.contains("�") {
error_tokens.push(next_token);
if error_tokens.len() > 3 {
error_tokens.clear();
}
seqlen_offset += seq_len;
seq_len = 1;
input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
let appendd_mask = Tensor::ones((1, 1), mask.dtype(), &self.device)?;
mask = Tensor::cat(&[mask, appendd_mask], 1)?;
cache_position = Tensor::from_vec(vec![seqlen_offset as u32], 1, &self.device)?;
pixel_values = None;
pixel_values_video = None;
continue;
}
error_tokens.clear();
let chunk = build_completion_chunk_response(decoded_token, "qwen2.5vl", None, None);
yield Ok(chunk);
if next_token == self.endoftext_id || next_token == self.im_end_id {
break;
}
seqlen_offset += seq_len;
seq_len = 1;
input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
let appendd_mask = Tensor::ones((1, 1), mask.dtype(), &self.device)?;
mask = Tensor::cat(&[mask, appendd_mask], 1)?;
cache_position = Tensor::from_vec(vec![seqlen_offset as u32], 1, &self.device)?;
pixel_values = None;
pixel_values_video = None;
}
self.qwen2_5_vl.clear_kv_cache();
};
Ok(stream)
}
}
src/utils/utils.rs
+172
View File
@@ -1,5 +1,12 @@
use anyhow::Result;
use candle_core::{DType, Device};
use candle_transformers::generation::LogitsProcessor;
use openai_dive::v1::resources::{
chat::{
ChatCompletionChoice, ChatCompletionChunkChoice, ChatCompletionChunkResponse, ChatCompletionResponse, ChatMessage, ChatMessageContent, DeltaChatMessage, DeltaFunction, DeltaToolCall, Function, ToolCall
},
shared::FinishReason,
};
pub fn get_device(device: Option<&Device>) -> Device {
match device {
@@ -85,3 +92,168 @@ pub fn ceil_by_factor(num: f32, factor: u32) -> u32 {
let ceil = (num / factor as f32).ceil() as u32;
ceil * factor
}
pub fn build_completion_response(res: String, model_name: &str) -> ChatCompletionResponse {
let id = uuid::Uuid::new_v4().to_string();
let mut response = ChatCompletionResponse {
id: Some(id),
choices: vec![],
created: chrono::Utc::now().timestamp() as u32,
model: model_name.to_string(),
service_tier: None,
system_fingerprint: None,
object: "chat.completion".to_string(),
usage: None,
};
let choice = if res.contains("<tool_call>") {
let mes: Vec<&str> = res.split("<tool_call>").collect();
let content = mes[0].to_string();
let mut tool_vec = Vec::new();
for i in 1..mes.len() {
let tool_mes = mes[i].replace("</tool_call>", "");
let function = match serde_json::from_str::<serde_json::Value>(&tool_mes) {
Ok(json_value) => {
let name = json_value
.get("name")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.unwrap_or_default();
let arguments = json_value
.get("arguments")
.map(|v| v.to_string())
.unwrap_or_default();
Function { name, arguments }
}
Err(_) => Function {
name: "".to_string(),
arguments: "".to_string(),
},
};
let tool_call = ToolCall {
id: (i - 1).to_string(),
r#type: "function".to_string(),
function: function,
};
tool_vec.push(tool_call);
}
ChatCompletionChoice {
index: 0,
message: ChatMessage::Assistant {
content: Some(ChatMessageContent::Text(content)),
reasoning_content: None,
refusal: None,
name: None,
audio: None,
tool_calls: Some(tool_vec),
},
finish_reason: Some(FinishReason::ToolCalls),
logprobs: None,
}
} else {
ChatCompletionChoice {
index: 0,
message: ChatMessage::Assistant {
content: Some(ChatMessageContent::Text(res)),
reasoning_content: None,
refusal: None,
name: None,
audio: None,
tool_calls: None,
},
finish_reason: Some(FinishReason::StopSequenceReached),
logprobs: None,
}
};
response.choices.push(choice);
response
}
pub fn build_completion_chunk_response(
res: String,
model_name: &str,
tool_call_id: Option<String>,
tool_call_content: Option<String>,
) -> ChatCompletionChunkResponse {
let id = uuid::Uuid::new_v4().to_string();
let mut response = ChatCompletionChunkResponse {
id: Some(id),
choices: vec![],
created: chrono::Utc::now().timestamp() as u32,
model: model_name.to_string(),
system_fingerprint: None,
object: "chat.completion.chunk".to_string(),
usage: None,
};
let choice = if tool_call_id.is_some() {
let tool_call_id = tool_call_id.unwrap();
let function = if let Some(content) = tool_call_content {
match serde_json::from_str::<serde_json::Value>(&content) {
Ok(json_value) => {
let name = json_value
.get("name")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let arguments = json_value.get("arguments").map(|v| v.to_string());
DeltaFunction { name, arguments }
}
Err(_) => DeltaFunction {
name: None,
arguments: Some(content),
},
}
} else {
DeltaFunction {
name: None,
arguments: None,
}
};
ChatCompletionChunkChoice {
index: Some(0),
delta: DeltaChatMessage::Assistant {
content: None,
reasoning_content: None,
refusal: None,
name: None,
tool_calls: Some(vec![DeltaToolCall {
index: Some(0),
id: Some(tool_call_id),
r#type: Some("function".to_string()),
function,
}]),
},
finish_reason: None,
logprobs: None,
}
} else {
ChatCompletionChunkChoice {
index: Some(0),
delta: DeltaChatMessage::Assistant {
content: Some(ChatMessageContent::Text(res)),
reasoning_content: None,
refusal: None,
name: None,
tool_calls: None,
},
finish_reason: None,
logprobs: None,
}
};
response.choices.push(choice);
response
}
pub fn get_logit_processor(temperature: Option<f32>, top_p: Option<f32>) -> LogitsProcessor {
let temperature = match temperature {
Some(temp) => Some(temp as f64),
None => None,
};
let top_p = match top_p {
Some(tp) => Some(tp as f64),
None => None,
};
LogitsProcessor::new(34562, temperature, top_p)
}
tests/test_qwen2_5vl.rs
+54 -2
View File
@@ -1,9 +1,10 @@
use std::time::Instant;
use std::{pin::pin, time::Instant};
use aha::{models::{qwen2_5vl::generate::Qwen2_5VLGenerateModel, GenerateModel}, ModelType};
use anyhow::{Result};
use candle_core::{DType, Device};
use openai_dive::v1::resources::chat::ChatCompletionParameters;
use rocket::futures::StreamExt;
#[test]
@@ -48,7 +49,58 @@ fn qwen2_5vl_generate() -> Result<()> {
let i_start = Instant::now();
let result = model.generate(mes)?;
println!("generate: \n{}", result);
println!("generate: \n {:?}", result);
let i_duration = i_start.elapsed();
println!("Time elapsed in generate is: {:?}", i_duration);
Ok(())
}
#[tokio::test]
async fn qwen2_5vl_stream() -> Result<()> {
// test with cuda+flash-attn: RUST_BACKTRACE=1 cargo test -F cuda,flash-attn qwen2_5vl_generate -- --nocapture
let device = Device::cuda_if_available(0)?;
let dtype = DType::BF16;
let model_path = "/home/jhq/huggingface_model/Qwen/Qwen2.5-VL-3B-Instruct/";
let message = r#"
{
"model": "qwen2.5vl",
"messages": [
{
"role": "user",
"content": [
{
"type": "image",
"image_url":
{
"url": "file://./assets/img/ocr_test.png"
}
},
{
"type": "text",
"text": "请分析图片并提取所有可见文本内容，按从左到右、从上到下的布局，返回纯文本"
}
]
}
]
}
"#;
let mes:ChatCompletionParameters = serde_json::from_str(message)?;
let i_start = Instant::now();
// let mut model = Qwen2_5VLGenerateModel::init(model_path, &device, dtype)?;
let mut model = ModelType::init(ModelType::Qwen2_5VL, model_path, None, None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
let i_start = Instant::now();
let mut stream = pin!(model.generate_stream(mes)?);
while let Some(item) = stream.next().await {
println!("generate: \n {:?}", item);
}
let i_duration = i_start.elapsed();
println!("Time elapsed in generate is: {:?}", i_duration);