diff --git a/Cargo.lock b/Cargo.lock index 58f18ce..9c7d449 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,7 +44,7 @@ dependencies = [ "num", "rayon", "realfft", - "reqwest 0.12.28", + "reqwest 0.13.2", "rocket", "sentencepiece", "serde", @@ -2805,9 +2805,9 @@ dependencies = [ [[package]] name = "modelscope" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0ce1b3f708ede609c998fc460d5dce1b8b9458f99d71715fc8735f8335d8f3" +checksum = "e70d3712581a1c6148246dc7cefd63d2dbb3a40ba70602170adbec34bb7319bf" dependencies = [ "anyhow", "clap", @@ -3810,21 +3810,16 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", - "encoding_rs", - "futures-channel", "futures-core", "futures-util", - "h2 0.4.13", "http 1.4.0", "http-body 1.0.1", "http-body-util", "hyper 1.8.1", - "hyper-rustls", "hyper-tls", "hyper-util", "js-sys", "log", - "mime", "mime_guess", "native-tls", "percent-encoding", @@ -3858,6 +3853,7 @@ dependencies = [ "cookie", "cookie_store", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2 0.4.13", diff --git a/Cargo.toml b/Cargo.toml index ab4596c..6db4f22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ serde_json = "1.0.145" anyhow = "1.0.100" ffmpeg-next = { version = "8.0.0", optional = true } image = "0.25.8" -reqwest = { version = "0.12.23", features = ["blocking"] } +reqwest = { version = "0.13", features = ["blocking"] } base64 = "0.22.1" num = "0.4.3" minijinja = "2.12.0" @@ -28,7 +28,7 @@ rocket = { version = "0.5.1", features = ["serde_json", "json"] } tokio = "1.47.1" hound = "3.5.1" clap = { version = "4.5.51", features = ["derive"] } -modelscope = "0.1.4" +modelscope = "0.1.5" dirs = "6.0.0" sysinfo = "0.33" url = "2.5.7" diff --git a/README.md b/README.md index fcc6932..d63a326 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,7 @@ Apache-2.0 — See [LICENSE](LICENSE) for details. - All model authors and contributors ## Wechat -![260326 expired](./assets/img/aha_weixingqun.jpg) +![260405 expired](./assets/img/aha_weixingqun.png) ---

diff --git a/README.zh-CN.md b/README.zh-CN.md index dbebe5b..c23ce81 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -218,6 +218,9 @@ Apache-2.0 — 详见 [LICENSE](LICENSE) - [Candle](https://github.com/huggingface/candle) - 优秀的 Rust 机器学习框架 - 所有模型作者和贡献者 +## Wechat +![260405 expired](./assets/img/aha_weixinqun.png) + ---

diff --git a/assets/img/aha_weixingqun.jpg b/assets/img/aha_weixingqun.jpg deleted file mode 100644 index e6f25db..0000000 Binary files a/assets/img/aha_weixingqun.jpg and /dev/null differ diff --git a/assets/img/aha_weixinqun.png b/assets/img/aha_weixinqun.png new file mode 100644 index 0000000..59a59b7 Binary files /dev/null and b/assets/img/aha_weixinqun.png differ diff --git a/src/api/mod.rs b/src/api/mod.rs index 4ade969..6cd72cc 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -249,8 +249,10 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str { WhichModel::MiniCPM4_0_5B => "minicpm4-0.5b", WhichModel::LFM2_1_2B => "lfm2-1.2b", WhichModel::LFM2_5_1_2BInstruct => "lfm2.5-1.2b-instruct", - WhichModel::Qwen2_5vl3B => "qwen2.5vl-3b", - WhichModel::Qwen2_5vl7B => "qwen2.5vl-7b", + WhichModel::LFM2_5VL1_6B => "lfm2.5-vl-1.6b", + WhichModel::LFM2VL1_6B => "lfm2-vl-1.6b", + WhichModel::Qwen2_5VL3B => "qwen2.5vl-3b", + WhichModel::Qwen2_5VL7B => "qwen2.5vl-7b", WhichModel::Qwen3_0_6B => "qwen3-0.6b", WhichModel::Qwen3_5_0_8B => "qwen3.5-0.8b", WhichModel::Qwen3_5_2B => "qwen3.5-2b", @@ -259,10 +261,10 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str { WhichModel::Qwen3_5Gguf => "qwen3.5-gguf", WhichModel::Qwen3ASR0_6B => "qwen3asr-0.6b", WhichModel::Qwen3ASR1_7B => "qwen3asr-1.7b", - WhichModel::Qwen3vl2B => "qwen3vl-2b", - WhichModel::Qwen3vl4B => "qwen3vl-4b", - WhichModel::Qwen3vl8B => "qwen3vl-8b", - WhichModel::Qwen3vl32B => "qwen3vl-32b", + WhichModel::Qwen3VL2B => "qwen3vl-2b", + WhichModel::Qwen3VL4B => "qwen3vl-4b", + WhichModel::Qwen3VL8B => "qwen3vl-8b", + WhichModel::Qwen3VL32B => "qwen3vl-32b", WhichModel::DeepSeekOCR => "deepseek-ocr", WhichModel::DeepSeekOCR2 => "deepseek-ocr2", WhichModel::HunyuanOCR => "hunyuan-ocr", @@ -281,12 +283,12 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str { fn which_model_to_owner(which_model: WhichModel) -> &'static str { match which_model { WhichModel::MiniCPM4_0_5B => "OpenBMB", - WhichModel::Qwen2_5vl3B | WhichModel::Qwen2_5vl7B => "Qwen", + WhichModel::Qwen2_5VL3B | WhichModel::Qwen2_5VL7B => "Qwen", WhichModel::Qwen3_0_6B | WhichModel::Qwen3ASR0_6B | WhichModel::Qwen3ASR1_7B => "Qwen", - WhichModel::Qwen3vl2B - | WhichModel::Qwen3vl4B - | WhichModel::Qwen3vl8B - | WhichModel::Qwen3vl32B + WhichModel::Qwen3VL2B + | WhichModel::Qwen3VL4B + | WhichModel::Qwen3VL8B + | WhichModel::Qwen3VL32B | WhichModel::Qwen3_5Gguf => "Qwen", WhichModel::Qwen3_5_0_8B | WhichModel::Qwen3_5_2B @@ -299,7 +301,10 @@ fn which_model_to_owner(which_model: WhichModel) -> &'static str { WhichModel::VoxCPM | WhichModel::VoxCPM1_5 => "OpenBMB", WhichModel::GlmASRNano2512 | WhichModel::GlmOCR => "ZhipuAI", WhichModel::FunASRNano2512 => "FunAudioLLM", - WhichModel::LFM2_1_2B | WhichModel::LFM2_5_1_2BInstruct => "LiquidAI", + WhichModel::LFM2_1_2B + | WhichModel::LFM2_5_1_2BInstruct + | WhichModel::LFM2_5VL1_6B + | WhichModel::LFM2VL1_6B => "LiquidAI", } } @@ -380,13 +385,13 @@ mod tests { #[test] fn test_get_model_type_llm() { assert_eq!(WhichModel::Qwen3_0_6B.model_type(), "llm"); - assert_eq!(WhichModel::Qwen3vl2B.model_type(), "llm"); + assert_eq!(WhichModel::Qwen3VL2B.model_type(), "llm"); assert_eq!(WhichModel::MiniCPM4_0_5B.model_type(), "llm"); - assert_eq!(WhichModel::Qwen2_5vl3B.model_type(), "llm"); - assert_eq!(WhichModel::Qwen2_5vl7B.model_type(), "llm"); - assert_eq!(WhichModel::Qwen3vl4B.model_type(), "llm"); - assert_eq!(WhichModel::Qwen3vl8B.model_type(), "llm"); - assert_eq!(WhichModel::Qwen3vl32B.model_type(), "llm"); + assert_eq!(WhichModel::Qwen2_5VL3B.model_type(), "llm"); + assert_eq!(WhichModel::Qwen2_5VL7B.model_type(), "llm"); + assert_eq!(WhichModel::Qwen3VL4B.model_type(), "llm"); + assert_eq!(WhichModel::Qwen3VL8B.model_type(), "llm"); + assert_eq!(WhichModel::Qwen3VL32B.model_type(), "llm"); } #[test] @@ -407,6 +412,10 @@ mod tests { #[test] fn test_get_model_type_image() { assert_eq!(WhichModel::RMBG2_0.model_type(), "image"); + } + + #[test] + fn test_get_model_type_tts() { assert_eq!(WhichModel::VoxCPM.model_type(), "image"); assert_eq!(WhichModel::VoxCPM1_5.model_type(), "image"); } diff --git a/src/exec/lfm2.rs b/src/exec/lfm2.rs index 41986ff..2c99099 100644 --- a/src/exec/lfm2.rs +++ b/src/exec/lfm2.rs @@ -1,4 +1,4 @@ -//! MiniCPM4-0.5B exec implementation for CLI `run` subcommand +//! LFM2 exec implementation for CLI `run` subcommand use std::time::Instant; diff --git a/src/exec/lfm2vl.rs b/src/exec/lfm2vl.rs new file mode 100644 index 0000000..53be6b9 --- /dev/null +++ b/src/exec/lfm2vl.rs @@ -0,0 +1,76 @@ +//! LFM2.5VL exec implementation for CLI `run` subcommand + +use std::time::Instant; + +use anyhow::{Ok, Result}; + +use crate::exec::ExecModel; +use crate::models::GenerateModel; +use crate::models::lfm2vl::generate::Lfm2VLGenerateModel; +use crate::utils::get_file_path; + +pub struct Lfm2VLExec; + +impl ExecModel for Lfm2VLExec { + fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> { + let input_text = &input[0]; + let target_text = if input_text.starts_with("file://") { + let path = get_file_path(input_text)?; + std::fs::read_to_string(path)? + } else { + input_text.clone() + }; + let url = &input[1]; + let input_url = if url.starts_with("http://") + || url.starts_with("https://") + || url.starts_with("file://") + { + url.clone() + } else { + format!("file://{}", url) + }; + let i_start = Instant::now(); + let mut model = Lfm2VLGenerateModel::init(weight_path, None, None)?; + let i_duration = i_start.elapsed(); + println!("Time elapsed in load model is: {:?}", i_duration); + + let message = format!( + r#"{{ + "model": "lfm2.5vl", + "messages": [ + {{ + "role": "user", + "content": [ + {{ + "type": "image", + "image_url": {{ + "url": "{}" + }} + }}, + {{ + "type": "text", + "text": "{}" + }} + ] + }} + ] + }}"#, + input_url, target_text + ); + let mes = serde_json::from_str(&message)?; + + let i_start = Instant::now(); + let result = model.generate(mes)?; + let i_duration = i_start.elapsed(); + println!("Time elapsed in generate is: {:?}", i_duration); + + println!("Result: {:?}", result); + + if let Some(out) = output { + std::fs::write(out, format!("{:?}", result))?; + println!("Output saved to: {}", out); + } + + Ok(()) + } +} diff --git a/src/exec/mod.rs b/src/exec/mod.rs index 0e59094..a2922c8 100644 --- a/src/exec/mod.rs +++ b/src/exec/mod.rs @@ -9,6 +9,7 @@ pub mod glm_asr_nano; pub mod glm_ocr; pub mod hunyuan_ocr; pub mod lfm2; +pub mod lfm2vl; pub mod minicpm4; pub mod paddleocr_vl; pub mod qwen2_5vl; diff --git a/src/exec/qwen2_5vl.rs b/src/exec/qwen2_5vl.rs index ecaaa2e..f374902 100644 --- a/src/exec/qwen2_5vl.rs +++ b/src/exec/qwen2_5vl.rs @@ -8,9 +8,9 @@ use crate::exec::ExecModel; use crate::models::{GenerateModel, qwen2_5vl::generate::Qwen2_5VLGenerateModel}; use crate::utils::get_file_path; -pub struct Qwen2_5vlExec; +pub struct Qwen2_5VLExec; -impl ExecModel for Qwen2_5vlExec { +impl ExecModel for Qwen2_5VLExec { fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> { let input_text = &input[0]; let target_text = if input_text.starts_with("file://") { diff --git a/src/exec/qwen3_asr.rs b/src/exec/qwen3_asr.rs index 2b6bde0..acd13f3 100644 --- a/src/exec/qwen3_asr.rs +++ b/src/exec/qwen3_asr.rs @@ -1,4 +1,4 @@ -//! Fun-ASR-Nano-2512 exec implementation for CLI `run` subcommand +//! Qwen3ASR exec implementation for CLI `run` subcommand use std::time::Instant; diff --git a/src/exec/qwen3vl.rs b/src/exec/qwen3vl.rs index 26c1c0f..7622047 100644 --- a/src/exec/qwen3vl.rs +++ b/src/exec/qwen3vl.rs @@ -8,9 +8,9 @@ use crate::exec::ExecModel; use crate::models::{GenerateModel, qwen3vl::generate::Qwen3VLGenerateModel}; use crate::utils::get_file_path; -pub struct Qwen3vlExec; +pub struct Qwen3VLExec; -impl ExecModel for Qwen3vlExec { +impl ExecModel for Qwen3VLExec { fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> { let input_text = &input[0]; let target_text = if input_text.starts_with("file://") { diff --git a/src/main.rs b/src/main.rs index 299d77a..19422b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -246,8 +246,8 @@ fn run_list(args: ListArgs) -> anyhow::Result<()> { WhichModel::MiniCPM4_0_5B, WhichModel::LFM2_1_2B, WhichModel::LFM2_5_1_2BInstruct, - WhichModel::Qwen2_5vl3B, - WhichModel::Qwen2_5vl7B, + WhichModel::Qwen2_5VL3B, + WhichModel::Qwen2_5VL7B, WhichModel::Qwen3_0_6B, WhichModel::Qwen3_5_0_8B, WhichModel::Qwen3_5_2B, @@ -255,10 +255,10 @@ fn run_list(args: ListArgs) -> anyhow::Result<()> { WhichModel::Qwen3_5_9B, WhichModel::Qwen3ASR0_6B, WhichModel::Qwen3ASR1_7B, - WhichModel::Qwen3vl2B, - WhichModel::Qwen3vl4B, - WhichModel::Qwen3vl8B, - WhichModel::Qwen3vl32B, + WhichModel::Qwen3VL2B, + WhichModel::Qwen3VL4B, + WhichModel::Qwen3VL8B, + WhichModel::Qwen3VL32B, WhichModel::DeepSeekOCR, WhichModel::DeepSeekOCR2, WhichModel::HunyuanOCR, @@ -471,13 +471,21 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> { use aha::exec::lfm2::Lfm2Exec; Lfm2Exec::run(&input, output.as_deref(), &weight_path)?; } - WhichModel::Qwen2_5vl3B => { - use aha::exec::qwen2_5vl::Qwen2_5vlExec; - Qwen2_5vlExec::run(&input, output.as_deref(), &weight_path)?; + WhichModel::LFM2_5VL1_6B => { + use aha::exec::lfm2vl::Lfm2VLExec; + Lfm2VLExec::run(&input, output.as_deref(), &weight_path)?; } - WhichModel::Qwen2_5vl7B => { - use aha::exec::qwen2_5vl::Qwen2_5vlExec; - Qwen2_5vlExec::run(&input, output.as_deref(), &weight_path)?; + WhichModel::LFM2VL1_6B => { + use aha::exec::lfm2vl::Lfm2VLExec; + Lfm2VLExec::run(&input, output.as_deref(), &weight_path)?; + } + WhichModel::Qwen2_5VL3B => { + use aha::exec::qwen2_5vl::Qwen2_5VLExec; + Qwen2_5VLExec::run(&input, output.as_deref(), &weight_path)?; + } + WhichModel::Qwen2_5VL7B => { + use aha::exec::qwen2_5vl::Qwen2_5VLExec; + Qwen2_5VLExec::run(&input, output.as_deref(), &weight_path)?; } WhichModel::Qwen3_0_6B => { use aha::exec::qwen3::Qwen3Exec; @@ -511,21 +519,21 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> { use aha::exec::qwen3_asr::Qwen3ASRExec; Qwen3ASRExec::run(&input, output.as_deref(), &weight_path)?; } - WhichModel::Qwen3vl2B => { - use aha::exec::qwen3vl::Qwen3vlExec; - Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?; + WhichModel::Qwen3VL2B => { + use aha::exec::qwen3vl::Qwen3VLExec; + Qwen3VLExec::run(&input, output.as_deref(), &weight_path)?; } - WhichModel::Qwen3vl4B => { - use aha::exec::qwen3vl::Qwen3vlExec; - Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?; + WhichModel::Qwen3VL4B => { + use aha::exec::qwen3vl::Qwen3VLExec; + Qwen3VLExec::run(&input, output.as_deref(), &weight_path)?; } - WhichModel::Qwen3vl8B => { - use aha::exec::qwen3vl::Qwen3vlExec; - Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?; + WhichModel::Qwen3VL8B => { + use aha::exec::qwen3vl::Qwen3VLExec; + Qwen3VLExec::run(&input, output.as_deref(), &weight_path)?; } - WhichModel::Qwen3vl32B => { - use aha::exec::qwen3vl::Qwen3vlExec; - Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?; + WhichModel::Qwen3VL32B => { + use aha::exec::qwen3vl::Qwen3VLExec; + Qwen3VLExec::run(&input, output.as_deref(), &weight_path)?; } WhichModel::DeepSeekOCR => { use aha::exec::deepseek_ocr::DeepSeekORExec; diff --git a/src/models/lfm2/model.rs b/src/models/lfm2/model.rs index 7e25a33..09070a6 100644 --- a/src/models/lfm2/model.rs +++ b/src/models/lfm2/model.rs @@ -197,7 +197,7 @@ impl Lfm2DecoderLayer { } pub struct Lfm2Decoder { - embed_tokens: Embedding, + pub embed_tokens: Embedding, layers: Vec, // rotary_emb: RoPE, pos_emb: RoPE, diff --git a/src/models/lfm2vl/config.rs b/src/models/lfm2vl/config.rs index 6d6872e..483c211 100644 --- a/src/models/lfm2vl/config.rs +++ b/src/models/lfm2vl/config.rs @@ -2,7 +2,6 @@ use candle_nn::Activation; use crate::models::lfm2::config::Lfm2Config; - #[derive(Debug, Clone, PartialEq, serde::Deserialize)] pub struct Lfm2VLConfig { pub do_image_splitting: bool, @@ -19,7 +18,7 @@ pub struct Lfm2VLConfig { pub projector_bias: bool, pub projector_hidden_act: Activation, pub projector_hidden_size: usize, - pub projector_use_layernorm: bool, + pub projector_use_layernorm: Option, pub text_config: Lfm2Config, pub tile_size: usize, pub use_image_special_tokens: bool, @@ -27,7 +26,6 @@ pub struct Lfm2VLConfig { pub vision_config: Lfm2VLVisionConfig, } - #[derive(Debug, Clone, PartialEq, serde::Deserialize)] pub struct Lfm2VLVisionConfig { pub attention_dropout: f64, @@ -37,8 +35,8 @@ pub struct Lfm2VLVisionConfig { pub intermediate_size: usize, pub layer_norm_eps: f64, pub model_type: String, - pub num_attention_heads: u32, - pub num_channels: u32, + pub num_attention_heads: usize, + pub num_channels: usize, pub num_hidden_layers: usize, pub num_patches: usize, pub patch_size: usize, @@ -79,4 +77,3 @@ pub struct Size { pub struct Lfm2ProcessorConfig { pub image_processor: Lfm2ImageConfig, } - diff --git a/src/models/lfm2vl/generate.rs b/src/models/lfm2vl/generate.rs index cc3ba78..8e32718 100644 --- a/src/models/lfm2vl/generate.rs +++ b/src/models/lfm2vl/generate.rs @@ -1,22 +1,28 @@ -use aha_openai_dive::v1::resources::chat::ChatCompletionParameters; +use aha_openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatCompletionResponse}; use anyhow::Result; -use candle_core::{DType, Device}; +use candle_core::{DType, Device, Tensor}; +use candle_nn::VarBuilder; use crate::{ chat_template::ChatTemplate, models::{ + GenerateModel, lfm2::config::Lfm2GenerateConfig, - lfm2vl::{config::Lfm2VLConfig, processor::Lfm2VLProcessor}, + lfm2vl::{config::Lfm2VLConfig, model::Lfm2VLModel, processor::Lfm2VLProcessor}, }, tokenizer::TokenizerModel, - utils::{find_type_files, get_device, get_dtype, get_logit_processor}, + utils::{ + build_completion_chunk_response, build_completion_response, find_type_files, get_device, + get_dtype, get_logit_processor, + }, }; +use rocket::async_stream::stream; pub struct Lfm2VLGenerateModel<'a> { chat_template: ChatTemplate<'a>, tokenizer: TokenizerModel, device: Device, - // model: Lfm2VLModel, + model: Lfm2VLModel, processor: Lfm2VLProcessor, eos_token_id: u32, model_name: String, @@ -33,8 +39,8 @@ impl<'a> Lfm2VLGenerateModel<'a> { let model_path = find_type_files(path, "safetensors")?; let dtype = get_dtype(dtype, &cfg.dtype); - // let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_path, dtype, &device)? }; - // let model = Lfm2Model::new(vb, &cfg)?; + let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_path, dtype, &device)? }; + let model = Lfm2VLModel::new(vb, &cfg)?; let processor = Lfm2VLProcessor::new(path, dtype, &device)?; let eos_token_id = gen_cfg.eos_token_id; let model_name = std::path::Path::new(path) @@ -46,14 +52,16 @@ impl<'a> Lfm2VLGenerateModel<'a> { chat_template, tokenizer, device, - // model, + model, processor, eos_token_id, model_name, }) } +} - pub fn generate(&mut self, mes: ChatCompletionParameters) -> Result<()> { +impl<'a> GenerateModel for Lfm2VLGenerateModel<'a> { + fn generate(&mut self, mes: ChatCompletionParameters) -> Result { let mes_render = self.chat_template.apply_chat_template(&mes)?; let mut logits = get_logit_processor( mes.temperature, @@ -63,11 +71,125 @@ impl<'a> Lfm2VLGenerateModel<'a> { ); let (pixel_values, pixel_attention_mask, spatial_shapes, text) = self.processor.process_info(&mes, &mes_render)?; - let input_ids = self.tokenizer.text_encode(text, &self.device)?; - println!("pixel_values: {}", pixel_values); - println!("pixel_attention_mask: {}", pixel_attention_mask); - println!("spatial_shapes: {}", spatial_shapes); - println!("input_ids: {}", input_ids); - Ok(()) + let mut input_ids = self.tokenizer.text_encode(text, &self.device)?; + let mut seq_len = input_ids.dim(1)?; + let prompt_tokens = seq_len as u32; + let mut seqlen_offset = 0; + let mut generate = vec![]; + let sample_len = mes.max_tokens.unwrap_or(1024); + let mut pixel_values = Some(pixel_values); + let mut pixel_attention_mask = Some(pixel_attention_mask); + let mut spatial_shapes = Some(spatial_shapes); + for _ in 0..sample_len { + let logit = self.model.forward( + &input_ids, + pixel_values.as_ref(), + pixel_attention_mask.as_ref(), + spatial_shapes.as_ref(), + seqlen_offset, + )?; + let logit = logit.squeeze(0)?.squeeze(0)?; + let next_token = logits.sample(&logit)?; + generate.push(next_token); + if next_token == self.eos_token_id { + break; + } + input_ids = Tensor::new(vec![next_token], &self.device)?.unsqueeze(0)?; + seqlen_offset += seq_len; + seq_len = 1; + pixel_values = None; + pixel_attention_mask = None; + spatial_shapes = None; + } + self.model.clear_cache(); + let completion_tokens = generate.len() as u32; + let decode = self.tokenizer.token_decode(generate)?; + let mes = build_completion_response( + decode, + &self.model_name, + Some(completion_tokens), + Some(prompt_tokens), + ); + Ok(mes) + } + + fn generate_stream( + &mut self, + mes: ChatCompletionParameters, + ) -> Result< + Box< + dyn rocket::futures::Stream< + Item = Result< + aha_openai_dive::v1::resources::chat::ChatCompletionChunkResponse, + anyhow::Error, + >, + > + Send + + Unpin + + '_, + >, + > { + let mes_render = self.chat_template.apply_chat_template(&mes)?; + let mut logits = get_logit_processor( + mes.temperature, + mes.top_p, + None, + mes.seed.unwrap_or(34562) as u64, + ); + let (pixel_values, pixel_attention_mask, spatial_shapes, text) = + self.processor.process_info(&mes, &mes_render)?; + let mut input_ids = self.tokenizer.text_encode(text, &self.device)?; + let mut seq_len = input_ids.dim(1)?; + let mut seqlen_offset = 0; + let sample_len = mes.max_tokens.unwrap_or(1024); + let mut pixel_values = Some(pixel_values); + let mut pixel_attention_mask = Some(pixel_attention_mask); + let mut spatial_shapes = Some(spatial_shapes); + let stream = stream! { + let mut err_tokens = vec![]; + for _ in 0..sample_len { + let logit = self.model.forward( + &input_ids, + pixel_values.as_ref(), + pixel_attention_mask.as_ref(), + spatial_shapes.as_ref(), + seqlen_offset, + )?; + let logit = logit.squeeze(0)?.squeeze(0)?; + let next_token = logits.sample(&logit)?; + let mut decode_ids = vec![]; + if !err_tokens.is_empty() { + decode_ids.extend_from_slice(&err_tokens); + } + decode_ids.push(next_token); + let decode = self.tokenizer.token_decode(decode_ids)?; + if decode.contains("�") { + err_tokens.push(next_token); + if err_tokens.len() > 3 { + err_tokens.clear(); + } + input_ids = Tensor::new(vec![next_token], &self.device)?.unsqueeze(0)?; + seqlen_offset += seq_len; + seq_len = 1; + pixel_values = None; + pixel_attention_mask = None; + spatial_shapes = None; + continue; + } + err_tokens.clear(); + let chunk = build_completion_chunk_response(decode, &self.model_name, None, None); + yield Ok(chunk); + if next_token == self.eos_token_id { + break; + } + input_ids = Tensor::new(vec![next_token], &self.device)?.unsqueeze(0)?; + seqlen_offset += seq_len; + seq_len = 1; + pixel_values = None; + pixel_attention_mask = None; + spatial_shapes = None; + } + self.model.clear_cache(); + }; + Ok(Box::new(Box::pin(stream))) } } diff --git a/src/models/lfm2vl/mod.rs b/src/models/lfm2vl/mod.rs index ce9b2a0..8b1baf7 100644 --- a/src/models/lfm2vl/mod.rs +++ b/src/models/lfm2vl/mod.rs @@ -1,4 +1,4 @@ pub mod config; pub mod generate; pub mod model; -pub mod processor; \ No newline at end of file +pub mod processor; diff --git a/src/models/lfm2vl/model.rs b/src/models/lfm2vl/model.rs index e69de29..1199e96 100644 --- a/src/models/lfm2vl/model.rs +++ b/src/models/lfm2vl/model.rs @@ -0,0 +1,325 @@ +use crate::{ + models::{ + common::{NaiveAttnTwoLinearMLPBlock, get_layer_norm}, + lfm2::model::Lfm2Decoder, + lfm2vl::config::{Lfm2VLConfig, Lfm2VLVisionConfig}, + }, + utils::{ + interpolate::interpolate_bilinear, + tensor_utils::{get_equal_mask, masked_scatter_dim0, prepare_mask}, + }, +}; +use anyhow::{Result, anyhow}; +use candle_core::{D, IndexOp, Tensor}; +use candle_nn::{Activation, LayerNorm, Linear, Module, VarBuilder, embedding, linear_b}; +use num::integer::Roots; + +pub struct Siglip2VisionEmbeddings { + // embed_dim: usize, + // patch_size: usize, + patch_embedding: Linear, + // position_embedding_size: usize, + // position_embedding: Embedding, + postitional_embeddings: Tensor, +} + +impl Siglip2VisionEmbeddings { + pub fn new(vb: VarBuilder, cfg: &Lfm2VLVisionConfig) -> Result { + let embed_dim = cfg.hidden_size; + let patch_size = cfg.patch_size; + let patch_embedding = linear_b( + cfg.num_channels * patch_size * patch_size, + embed_dim, + true, + vb.pp("patch_embedding"), + )?; + let position_embedding_size = cfg.num_patches.sqrt(); + let position_embedding = + embedding(cfg.num_patches, embed_dim, vb.pp("position_embedding"))?; + // (num_patches, embed_dim) -> (h, w, embed_dim) + // ->(embed_dim, h, w) -> (1, embe_dim, h, w) + let postitional_embeddings = position_embedding + .embeddings() + .reshape((position_embedding_size, position_embedding_size, ()))? + .permute((2, 0, 1))? + .unsqueeze(0)?; + Ok(Self { + // embed_dim, + // patch_size, + patch_embedding, + // position_embedding_size, + // position_embedding, + postitional_embeddings, + }) + } + + fn resize_positional_embeddings( + &self, + spatial_shapes: &Tensor, + max_length: usize, + ) -> Result { + let mut result_pos_embeddings = vec![]; + let bs = spatial_shapes.dim(0)?; + for i in 0..bs { + let shape_i = spatial_shapes.i(i)?.to_vec1::()?; + let height = *shape_i.first().unwrap_or(&32) as usize; + let width = *shape_i.get(1).unwrap_or(&32) as usize; + + if height == 0 || width == 0 || height * width > max_length { + return Err(anyhow!("img height or width illigal")); + } + // (1, dim, height, width) -> (height*width, dim) + let resize_embeddings = interpolate_bilinear( + &self.postitional_embeddings, + (height, width), + Some(false), + Some(true), + )? + .reshape(((), height * width))? + .transpose(0, 1)?; + let resize_embeddings = if height * width < max_length { + let pad = max_length - height * width; + let pad_embedding = resize_embeddings.i(0)?.unsqueeze(0)?.repeat((pad, 1))?; + Tensor::cat(&[&resize_embeddings, &pad_embedding], 0)? + } else { + resize_embeddings + }; + result_pos_embeddings.push(resize_embeddings); + } + + let result_pos_embeddings = Tensor::stack(&result_pos_embeddings, 0)?; + Ok(result_pos_embeddings) + } + + pub fn forward(&self, pixel_values: &Tensor, spatial_shapes: &Tensor) -> Result { + let patch_embeds = self.patch_embedding.forward(pixel_values)?; + let max_length = pixel_values.dim(1)?; + let resize_pos_embeddings = + self.resize_positional_embeddings(spatial_shapes, max_length)?; + let embedding = patch_embeds.add(&resize_pos_embeddings)?; + Ok(embedding) + } +} + +pub struct Siglip2Encoder { + layers: Vec, +} + +impl Siglip2Encoder { + pub fn new(vb: VarBuilder, cfg: &Lfm2VLVisionConfig) -> Result { + let vb_layers = vb.pp("layers"); + let mut layers = vec![]; + for i in 0..cfg.num_hidden_layers { + let layer = NaiveAttnTwoLinearMLPBlock::new( + vb_layers.pp(i), + cfg.hidden_size, + cfg.num_attention_heads, + None, + None, + true, + "self_attn", + Some("out_proj"), + cfg.intermediate_size, + cfg.hidden_act, + true, + "mlp", + "fc1", + "fc2", + cfg.layer_norm_eps, + "layer_norm1", + "layer_norm2", + )?; + layers.push(layer); + } + Ok(Self { layers }) + } + + pub fn forward(&self, xs: &Tensor, attention_mask: Option<&Tensor>) -> Result { + let mut xs = xs.clone(); + for layer in &self.layers { + xs = layer.forward(&xs, None, None, attention_mask, false)?; + } + Ok(xs) + } +} + +pub struct Siglip2VisionModel { + embeddings: Siglip2VisionEmbeddings, + encoder: Siglip2Encoder, + post_layernorm: LayerNorm, +} + +impl Siglip2VisionModel { + pub fn new(vb: VarBuilder, cfg: &Lfm2VLVisionConfig) -> Result { + let vb = vb.pp("vision_model"); + let embeddings = Siglip2VisionEmbeddings::new(vb.pp("embeddings"), cfg)?; + let encoder = Siglip2Encoder::new(vb.pp("encoder"), cfg)?; + let post_layernorm = get_layer_norm( + vb.pp("post_layernorm"), + cfg.layer_norm_eps, + cfg.hidden_size, + true, + )?; + Ok(Self { + embeddings, + encoder, + post_layernorm, + }) + } + + pub fn forward( + &self, + pixel_values: &Tensor, + attention_mask: &Tensor, + spatial_shapes: &Tensor, + ) -> Result { + let xs = self.embeddings.forward(pixel_values, spatial_shapes)?; + let mask = prepare_mask(attention_mask)?.to_dtype(xs.dtype())?; + let xs = self.encoder.forward(&xs, Some(&mask))?; + let xs = self.post_layernorm.forward(&xs)?; + Ok(xs) + } +} + +pub struct Lfm2VlMultiModalProjector { + factor: usize, + layer_norm: Option, + linear_1: Linear, + act: Activation, + linear_2: Linear, +} + +impl Lfm2VlMultiModalProjector { + pub fn new(vb: VarBuilder, cfg: &Lfm2VLConfig) -> Result { + let in_channels = cfg.vision_config.hidden_size * (cfg.downsample_factor).pow(2); + let factor = cfg.downsample_factor; + let layer_norm = if let Some(flag) = cfg.projector_use_layernorm + && !flag + { + None + } else { + let layer_norm = get_layer_norm( + vb.pp("layer_norm"), + cfg.vision_config.layer_norm_eps, + in_channels, + true, + )?; + Some(layer_norm) + }; + let linear_1 = linear_b( + in_channels, + cfg.projector_hidden_size, + cfg.projector_bias, + vb.pp("linear_1"), + )?; + let act = cfg.projector_hidden_act; + let linear_2 = linear_b( + cfg.projector_hidden_size, + cfg.text_config.hidden_size, + cfg.projector_bias, + vb.pp("linear_2"), + )?; + Ok(Self { + factor, + layer_norm, + linear_1, + act, + linear_2, + }) + } + + pub fn forward(&self, xs: &Tensor) -> Result { + let (bs, w, h, c) = xs.dims4()?; + let xs = xs.reshape((bs, w, h / self.factor, c * self.factor))?; + let xs = xs.permute((0, 2, 1, 3))?; + let xs = xs.reshape(( + bs, + h / self.factor, + w / self.factor, + c * self.factor * self.factor, + ))?; + let mut xs = xs.permute((0, 2, 1, 3))?.contiguous()?; + if let Some(norm) = &self.layer_norm { + xs = norm.forward(&xs)?; + } + xs = self.linear_1.forward(&xs)?.apply(&self.act)?; + xs = self.linear_2.forward(&xs)?; + Ok(xs) + } +} + +pub struct Lfm2VLModel { + vision_tower: Siglip2VisionModel, + multi_modal_projector: Lfm2VlMultiModalProjector, + language_model: Lfm2Decoder, + lm_head: Linear, + img_id: u32, +} + +impl Lfm2VLModel { + pub fn new(vb: VarBuilder, cfg: &Lfm2VLConfig) -> Result { + let vb = vb.pp("model"); + let vision_tower = Siglip2VisionModel::new(vb.pp("vision_tower"), &cfg.vision_config)?; + let multi_modal_projector = + Lfm2VlMultiModalProjector::new(vb.pp("multi_modal_projector"), cfg)?; + let language_model = Lfm2Decoder::new(vb.pp("language_model"), &cfg.text_config)?; + let lm_head = Linear::new(language_model.embed_tokens.embeddings().clone(), None); + Ok(Self { + vision_tower, + multi_modal_projector, + language_model, + lm_head, + img_id: cfg.image_token_id, + }) + } + + pub fn forward( + &mut self, + input_ids: &Tensor, + pixel_values: Option<&Tensor>, + pixel_attention_mask: Option<&Tensor>, + spatial_shapes: Option<&Tensor>, + seqlen_offset: usize, + ) -> Result { + let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?; + if let Some(pixel) = pixel_values + && let Some(mask) = pixel_attention_mask + && let Some(shapes) = spatial_shapes + { + let image_embeds = self.vision_tower.forward(pixel, mask, shapes)?; + println!("image_embeds: {}", image_embeds); + let bs = image_embeds.dim(0)?; + let img_featrure_length = mask.sum(1)?.to_vec1::()?; + let mut image_features = vec![]; + for img_idx in 0..bs { + let feature = image_embeds.i(img_idx)?; + let feature = feature.narrow(0, 0, img_featrure_length[img_idx] as usize)?; + let shape = shapes.i(img_idx)?.to_vec1::()?; + let h = shape[0]; + let w = shape[1]; + let feature = feature + .reshape((1, h as usize, w as usize, ()))? + .contiguous()?; + let img_embedding = self.multi_modal_projector.forward(&feature)?; + let dim = img_embedding.dim(D::Minus1)?; + let img_embedding = img_embedding.reshape(((), dim))?; + image_features.push(img_embedding); + } + let image_embeds = Tensor::cat(&image_features, 0)?; + println!("image_embeds: {}", image_embeds); + let image_mask = get_equal_mask(input_ids, self.img_id)?; + inputs_embeds = masked_scatter_dim0(&inputs_embeds, &image_embeds, &image_mask)?; + } + let output = self + .language_model + .forward(input_ids, Some(&inputs_embeds), seqlen_offset)?; + let seq_len = output.dim(1)?; + let last = output.narrow(1, seq_len - 1, 1)?; + let logits = self.lm_head.forward(&last)?; + Ok(logits) + } + + pub fn clear_cache(&mut self) { + self.language_model.clear_cache(); + } +} diff --git a/src/models/lfm2vl/processor.rs b/src/models/lfm2vl/processor.rs index 327020e..74236dd 100644 --- a/src/models/lfm2vl/processor.rs +++ b/src/models/lfm2vl/processor.rs @@ -33,17 +33,24 @@ pub struct Lfm2VLProcessor { image_thumbnail_token: String, } +#[allow(clippy::type_complexity)] impl Lfm2VLProcessor { pub fn new(path: &str, dtype: DType, device: &Device) -> Result { - let path = path.to_string(); assert!( - std::path::Path::new(&path).exists(), + std::path::Path::new(path).exists(), "model path file not exists" ); - let processor_cfg_path = path + "/processor_config.json"; - let processor_cfg: Lfm2ProcessorConfig = - serde_json::from_slice(&std::fs::read(processor_cfg_path)?)?; - let image_config = processor_cfg.image_processor; + let processor_cfg_path = path.to_string() + "/processor_config.json"; + let processor_cfg = + serde_json::from_slice::(&std::fs::read(processor_cfg_path)?); + + let image_config = match processor_cfg { + Ok(cfg) => cfg.image_processor, + Err(_) => { + let processor_cfg_path = path.to_string() + "/preprocessor_config.json"; + serde_json::from_slice::(&std::fs::read(processor_cfg_path)?)? + } + }; // 256 let max_thumbnail_image_patches = image_config.max_image_tokens * image_config.downsample_factor.pow(2); @@ -160,7 +167,7 @@ impl Lfm2VLProcessor { let (new_height, new_width) = img_smart_resize( height, width, - self.total_factor as u32, + self.total_factor, self.smart_resize_min_pixels as u32, self.smart_resize_max_pixels as u32, )?; @@ -331,7 +338,6 @@ impl Lfm2VLProcessor { ) = self.process_imgs(imgs)?; let text = self.expand_text_with_placeholders(text, num_cols_list, num_rows_list, image_size_list); - Ok((pixel_values, pixel_attention_mask, spatial_shapes, text)) } } diff --git a/src/models/mod.rs b/src/models/mod.rs index c15debb..200780d 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -33,70 +33,74 @@ use crate::models::{ fun_asr_nano::generate::FunAsrNanoGenerateModel, glm_asr_nano::generate::GlmAsrNanoGenerateModel, glm_ocr::generate::GlmOcrGenerateModel, hunyuan_ocr::generate::HunyuanOCRGenerateModel, lfm2::generate::Lfm2GenerateModel, - minicpm4::generate::MiniCPMGenerateModel, paddleocr_vl::generate::PaddleOCRVLGenerateModel, - qwen2_5vl::generate::Qwen2_5VLGenerateModel, qwen3::generate::Qwen3GenerateModel, - qwen3_5::generate::Qwen3_5GenerateModel, qwen3_asr::generate::Qwen3AsrGenerateModel, - qwen3vl::generate::Qwen3VLGenerateModel, rmbg2_0::generate::RMBG2_0Model, - voxcpm::generate::VoxCPMGenerate, + lfm2vl::generate::Lfm2VLGenerateModel, minicpm4::generate::MiniCPMGenerateModel, + paddleocr_vl::generate::PaddleOCRVLGenerateModel, qwen2_5vl::generate::Qwen2_5VLGenerateModel, + qwen3::generate::Qwen3GenerateModel, qwen3_5::generate::Qwen3_5GenerateModel, + qwen3_asr::generate::Qwen3AsrGenerateModel, qwen3vl::generate::Qwen3VLGenerateModel, + rmbg2_0::generate::RMBG2_0Model, voxcpm::generate::VoxCPMGenerate, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] pub enum WhichModel { - #[value(name = "lfm2-1.2b", hide = true)] + #[value(name = "lfm2-1.2b")] LFM2_1_2B, - #[value(name = "lfm2.5-1.2b-instruct", hide = true)] + #[value(name = "lfm2.5-1.2b-instruct")] LFM2_5_1_2BInstruct, - #[value(name = "minicpm4-0.5b", hide = true)] + #[value(name = "lfm2.5-vl-1.6b")] + LFM2_5VL1_6B, + #[value(name = "lfm2-vl-1.6b")] + LFM2VL1_6B, + #[value(name = "minicpm4-0.5b")] MiniCPM4_0_5B, - #[value(name = "qwen2.5vl-3b", hide = true)] - Qwen2_5vl3B, - #[value(name = "qwen2.5vl-7b", hide = true)] - Qwen2_5vl7B, - #[value(name = "qwen3-0.6b", hide = true)] + #[value(name = "qwen2.5vl-3b")] + Qwen2_5VL3B, + #[value(name = "qwen2.5vl-7b")] + Qwen2_5VL7B, + #[value(name = "qwen3-0.6b")] Qwen3_0_6B, - #[value(name = "qwen3.5-0.8b", hide = true)] + #[value(name = "qwen3.5-0.8b")] Qwen3_5_0_8B, - #[value(name = "qwen3.5-2b", hide = true)] + #[value(name = "qwen3.5-2b")] Qwen3_5_2B, - #[value(name = "qwen3.5-4b", hide = true)] + #[value(name = "qwen3.5-4b")] Qwen3_5_4B, - #[value(name = "qwen3.5-9b", hide = true)] + #[value(name = "qwen3.5-9b")] Qwen3_5_9B, - #[value(name = "qwen3.5-gguf", hide = true)] + #[value(name = "qwen3.5-gguf")] Qwen3_5Gguf, - #[value(name = "qwen3asr-0.6b", hide = true)] + #[value(name = "qwen3asr-0.6b")] Qwen3ASR0_6B, - #[value(name = "qwen3asr-1.7b", hide = true)] + #[value(name = "qwen3asr-1.7b")] Qwen3ASR1_7B, - #[value(name = "qwen3vl-2b", hide = true)] - Qwen3vl2B, - #[value(name = "qwen3vl-4b", hide = true)] - Qwen3vl4B, - #[value(name = "qwen3vl-8b", hide = true)] - Qwen3vl8B, - #[value(name = "qwen3vl-32b", hide = true)] - Qwen3vl32B, - #[value(name = "deepseek-ocr", hide = true)] + #[value(name = "qwen3vl-2b")] + Qwen3VL2B, + #[value(name = "qwen3vl-4b")] + Qwen3VL4B, + #[value(name = "qwen3vl-8b")] + Qwen3VL8B, + #[value(name = "qwen3vl-32b")] + Qwen3VL32B, + #[value(name = "deepseek-ocr")] DeepSeekOCR, - #[value(name = "deepseek-ocr2", hide = true)] + #[value(name = "deepseek-ocr2")] DeepSeekOCR2, - #[value(name = "hunyuan-ocr", hide = true)] + #[value(name = "hunyuan-ocr")] HunyuanOCR, - #[value(name = "paddleocr-vl", hide = true)] + #[value(name = "paddleocr-vl")] PaddleOCRVL, - #[value(name = "paddleocr-vl1.5", hide = true)] + #[value(name = "paddleocr-vl1.5")] PaddleOCRVL1_5, #[value(name = "rmbg2.0")] RMBG2_0, - #[value(name = "voxcpm", hide = true)] + #[value(name = "voxcpm")] VoxCPM, - #[value(name = "voxcpm1.5", hide = true)] + #[value(name = "voxcpm1.5")] VoxCPM1_5, - #[value(name = "glm-asr-nano-2512", hide = true)] + #[value(name = "glm-asr-nano-2512")] GlmASRNano2512, - #[value(name = "fun-asr-nano-2512", hide = true)] + #[value(name = "fun-asr-nano-2512")] FunASRNano2512, - #[value(name = "glm-ocr", hide = true)] + #[value(name = "glm-ocr")] GlmOCR, } @@ -106,9 +110,11 @@ impl WhichModel { match self { WhichModel::LFM2_1_2B => "LiquidAI/LFM2-1.2B", WhichModel::LFM2_5_1_2BInstruct => "LiquidAI/LFM2.5-1.2B-Instruct", + WhichModel::LFM2_5VL1_6B => "LiquidAI/LFM2.5-VL-1.6B", + WhichModel::LFM2VL1_6B => "LiquidAI/LFM2-VL-1.6B", WhichModel::MiniCPM4_0_5B => "OpenBMB/MiniCPM4-0.5B", - WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct", - WhichModel::Qwen2_5vl7B => "Qwen/Qwen2.5-VL-7B-Instruct", + WhichModel::Qwen2_5VL3B => "Qwen/Qwen2.5-VL-3B-Instruct", + WhichModel::Qwen2_5VL7B => "Qwen/Qwen2.5-VL-7B-Instruct", WhichModel::Qwen3_0_6B => "Qwen/Qwen3-0.6B", WhichModel::Qwen3_5_0_8B => "Qwen/Qwen3.5-0.8B", WhichModel::Qwen3_5_2B => "Qwen/Qwen3.5-2B", @@ -117,10 +123,10 @@ impl WhichModel { WhichModel::Qwen3_5Gguf => "GGUF", WhichModel::Qwen3ASR0_6B => "Qwen/Qwen3-ASR-0.6B", WhichModel::Qwen3ASR1_7B => "Qwen/Qwen3-ASR-1.7B", - WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct", - WhichModel::Qwen3vl4B => "Qwen/Qwen3-VL-4B-Instruct", - WhichModel::Qwen3vl8B => "Qwen/Qwen3-VL-8B-Instruct", - WhichModel::Qwen3vl32B => "Qwen/Qwen3-VL-32B-Instruct", + WhichModel::Qwen3VL2B => "Qwen/Qwen3-VL-2B-Instruct", + WhichModel::Qwen3VL4B => "Qwen/Qwen3-VL-4B-Instruct", + WhichModel::Qwen3VL8B => "Qwen/Qwen3-VL-8B-Instruct", + WhichModel::Qwen3VL32B => "Qwen/Qwen3-VL-32B-Instruct", WhichModel::DeepSeekOCR => "deepseek-ai/DeepSeek-OCR", WhichModel::DeepSeekOCR2 => "deepseek-ai/DeepSeek-OCR-2", WhichModel::HunyuanOCR => "Tencent-Hunyuan/HunyuanOCR", @@ -143,17 +149,19 @@ impl WhichModel { | WhichModel::Qwen3_0_6B | WhichModel::LFM2_1_2B | WhichModel::LFM2_5_1_2BInstruct => "llm", - WhichModel::Qwen2_5vl3B - | WhichModel::Qwen2_5vl7B - | WhichModel::Qwen3vl2B - | WhichModel::Qwen3vl4B - | WhichModel::Qwen3vl8B - | WhichModel::Qwen3vl32B + WhichModel::Qwen2_5VL3B + | WhichModel::Qwen2_5VL7B + | WhichModel::Qwen3VL2B + | WhichModel::Qwen3VL4B + | WhichModel::Qwen3VL8B + | WhichModel::Qwen3VL32B | WhichModel::Qwen3_5_0_8B | WhichModel::Qwen3_5_2B | WhichModel::Qwen3_5_4B | WhichModel::Qwen3_5_9B - | WhichModel::Qwen3_5Gguf => "vlm", + | WhichModel::Qwen3_5Gguf + | WhichModel::LFM2_5VL1_6B + | WhichModel::LFM2VL1_6B => "vlm", // OCR models WhichModel::DeepSeekOCR | WhichModel::DeepSeekOCR2 @@ -167,7 +175,8 @@ impl WhichModel { | WhichModel::GlmASRNano2512 | WhichModel::FunASRNano2512 => "asr", // Image models - WhichModel::RMBG2_0 | WhichModel::VoxCPM | WhichModel::VoxCPM1_5 => "image", + WhichModel::RMBG2_0 => "image", + WhichModel::VoxCPM | WhichModel::VoxCPM1_5 => "tts", } } } @@ -190,6 +199,7 @@ pub trait GenerateModel { pub enum ModelInstance<'a> { MiniCPM4(MiniCPMGenerateModel<'a>), Lfm2(Lfm2GenerateModel<'a>), + Lfm2VL(Lfm2VLGenerateModel<'a>), Qwen2_5VL(Qwen2_5VLGenerateModel<'a>), Qwen3(Qwen3GenerateModel<'a>), Qwen3_5(Qwen3_5GenerateModel<'a>), @@ -210,6 +220,7 @@ impl<'a> GenerateModel for ModelInstance<'a> { match self { ModelInstance::MiniCPM4(model) => model.generate(mes), ModelInstance::Lfm2(model) => model.generate(mes), + ModelInstance::Lfm2VL(model) => model.generate(mes), ModelInstance::Qwen2_5VL(model) => model.generate(mes), ModelInstance::Qwen3(model) => model.generate(mes), ModelInstance::Qwen3_5(model) => model.generate(mes), @@ -240,6 +251,7 @@ impl<'a> GenerateModel for ModelInstance<'a> { match self { ModelInstance::MiniCPM4(model) => model.generate_stream(mes), ModelInstance::Lfm2(model) => model.generate_stream(mes), + ModelInstance::Lfm2VL(model) => model.generate_stream(mes), ModelInstance::Qwen2_5VL(model) => model.generate_stream(mes), ModelInstance::Qwen3(model) => model.generate_stream(mes), ModelInstance::Qwen3_5(model) => model.generate_stream(mes), @@ -276,11 +288,19 @@ pub fn load_model<'a>( let model = Lfm2GenerateModel::init(path, None, None)?; ModelInstance::Lfm2(model) } - WhichModel::Qwen2_5vl3B => { + WhichModel::LFM2_5VL1_6B => { + let model = Lfm2VLGenerateModel::init(path, None, None)?; + ModelInstance::Lfm2VL(model) + } + WhichModel::LFM2VL1_6B => { + let model = Lfm2VLGenerateModel::init(path, None, None)?; + ModelInstance::Lfm2VL(model) + } + WhichModel::Qwen2_5VL3B => { let model = Qwen2_5VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen2_5VL(model) } - WhichModel::Qwen2_5vl7B => { + WhichModel::Qwen2_5VL7B => { let model = Qwen2_5VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen2_5VL(model) } @@ -320,19 +340,19 @@ pub fn load_model<'a>( let model = Qwen3AsrGenerateModel::init(path, None, None)?; ModelInstance::Qwen3ASR(model) } - WhichModel::Qwen3vl2B => { + WhichModel::Qwen3VL2B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(Box::new(model)) } - WhichModel::Qwen3vl4B => { + WhichModel::Qwen3VL4B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(Box::new(model)) } - WhichModel::Qwen3vl8B => { + WhichModel::Qwen3VL8B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(Box::new(model)) } - WhichModel::Qwen3vl32B => { + WhichModel::Qwen3VL32B => { let model = Qwen3VLGenerateModel::init(path, None, None)?; ModelInstance::Qwen3VL(Box::new(model)) } diff --git a/src/utils/tensor_utils.rs b/src/utils/tensor_utils.rs index fbf2953..e491af3 100644 --- a/src/utils/tensor_utils.rs +++ b/src/utils/tensor_utils.rs @@ -1,3 +1,5 @@ +use std::f32; + use anyhow::{Result, anyhow}; use candle_core::{D, DType, Device, IndexOp, Tensor, shape::Dim}; use candle_nn::ops::sigmoid; @@ -32,6 +34,26 @@ pub fn attn_masked_fill(on_true: &Tensor, mask: &Tensor, on_false: f32) -> Resul Ok(filled) } +pub fn prepare_mask(mask: &Tensor) -> Result { + //(bs, seq_len) + // [[1, 1, 1, 1, 0, 0]] + // -> + // [[1, 1, 1, 1, 0, 0], + // [1, 1, 1, 1, 0, 0], + // [1, 1, 1, 1, 0, 0], + // [1, 1, 1, 1, 0, 0], + // [1, 1, 1, 1, 0, 0], + // [1, 1, 1, 1, 0, 0],] + // (bs, 1, 1, seq_len) + let seq_len = mask.dim(1)?; + let mask = mask.unsqueeze(1)?.unsqueeze(1)?; + let mask = mask.repeat((1, 1, seq_len, 1))?; + let on_true = mask.zeros_like()?.to_dtype(DType::F32)?; + let on_false = Tensor::new(f32::NEG_INFINITY, mask.device())?.broadcast_as(mask.shape())?; + let mask = mask.where_cond(&on_true, &on_false)?; + Ok(mask) +} + pub fn prepare_causal_attention_mask( b_size: usize, tgt_len: usize, diff --git a/tests/config_tests.rs b/tests/config_tests.rs index 62ca94b..b13fdd6 100644 --- a/tests/config_tests.rs +++ b/tests/config_tests.rs @@ -1,5 +1,13 @@ use aha::models::{ - deepseek_ocr::config::DeepseekOCRConfig, hunyuan_ocr::config::HunYuanVLConfig, lfm2::config::Lfm2Config, lfm2vl::config::{Lfm2ProcessorConfig, Lfm2VLConfig}, minicpm4::config::MiniCPM4Config, paddleocr_vl::config::PaddleOCRVLConfig, qwen2_5vl::config::Qwen2_5VLConfig, qwen3vl::config::Qwen3VLConfig, voxcpm::config::VoxCPMConfig + deepseek_ocr::config::DeepseekOCRConfig, + hunyuan_ocr::config::HunYuanVLConfig, + lfm2::config::Lfm2Config, + lfm2vl::config::{Lfm2ProcessorConfig, Lfm2VLConfig}, + minicpm4::config::MiniCPM4Config, + paddleocr_vl::config::PaddleOCRVLConfig, + qwen2_5vl::config::Qwen2_5VLConfig, + qwen3vl::config::Qwen3VLConfig, + voxcpm::config::VoxCPMConfig, }; use anyhow::Result; @@ -104,7 +112,8 @@ fn lfm2vl_config() -> Result<()> { let config: Lfm2VLConfig = serde_json::from_slice(&std::fs::read(config_path)?)?; println!("{:?}", config); let processor_config_path = model_path.to_string() + "/processor_config.json"; - let processor_config: Lfm2ProcessorConfig = serde_json::from_slice(&std::fs::read(processor_config_path)?)?; + let processor_config: Lfm2ProcessorConfig = + serde_json::from_slice(&std::fs::read(processor_config_path)?)?; println!("{:?}", processor_config); Ok(()) -} \ No newline at end of file +} diff --git a/tests/test_health_models.rs b/tests/test_health_models.rs index ed0e334..1e3b1eb 100644 --- a/tests/test_health_models.rs +++ b/tests/test_health_models.rs @@ -8,8 +8,8 @@ fn test_model_type_classification() { // Since get_model_type and get_model_id are private to api.rs, // we document the expected behavior here for reference: // - // LLM models: MiniCPM4_0_5B, Qwen2_5vl3B, Qwen2_5vl7B, Qwen3_0_6B, - // Qwen3vl2B, Qwen3vl4B, Qwen3vl8B, Qwen3vl32B + // LLM models: MiniCPM4_0_5B, Qwen2_5VL3B, Qwen2_5VL7B, Qwen3_0_6B, + // Qwen3VL2B, Qwen3VL4B, Qwen3VL8B, Qwen3VL32B // OCR models: DeepSeekOCR, HunyuanOCR, PaddleOCRVL // ASR models: Qwen3ASR0_6B, Qwen3ASR1_7B, GlmASRNano2512, FunASRNano2512 // Image models: RMBG2_0, VoxCPM, VoxCPM1_5 @@ -17,13 +17,13 @@ fn test_model_type_classification() { // This test documents the expected model type classification let llm_models = [ WhichModel::MiniCPM4_0_5B, - WhichModel::Qwen2_5vl3B, - WhichModel::Qwen2_5vl7B, + WhichModel::Qwen2_5VL3B, + WhichModel::Qwen2_5VL7B, WhichModel::Qwen3_0_6B, - WhichModel::Qwen3vl2B, - WhichModel::Qwen3vl4B, - WhichModel::Qwen3vl8B, - WhichModel::Qwen3vl32B, + WhichModel::Qwen3VL2B, + WhichModel::Qwen3VL4B, + WhichModel::Qwen3VL8B, + WhichModel::Qwen3VL32B, ]; let ocr_models = [ diff --git a/tests/test_lfm2.rs b/tests/test_lfm2.rs index aea053a..08cff28 100644 --- a/tests/test_lfm2.rs +++ b/tests/test_lfm2.rs @@ -9,19 +9,18 @@ use std::{pin::pin, time::Instant}; #[test] fn lfm2_generate() -> Result<()> { // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_lfm2 lfm2_generate -r -- --nocapture - // test with cuda+flash-attn: RUST_BACKTRACE=1 cargo test -F cuda,flash-attn qwen3_0_6b_generate -r -- --nocapture let save_dir = aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; - // let model_path = format!("{}/LiquidAI/LFM2-1.2B/", save_dir); - let model_path = format!("{}/LiquidAI/LFM2.5-1.2B-Instruct/", save_dir); + let model_path = format!("{}/LiquidAI/LFM2-1.2B/", save_dir); + // let model_path = format!("{}/LiquidAI/LFM2.5-1.2B-Instruct/", save_dir); let message = r#" { "model": "lfm2", "messages": [ { "role": "user", - "content": "你如何看待AI" + "content": "你是谁，你如何看待AI" } ] } diff --git a/tests/test_lfm2vl.rs b/tests/test_lfm2vl.rs index 9665d5d..22bbbb1 100644 --- a/tests/test_lfm2vl.rs +++ b/tests/test_lfm2vl.rs @@ -1,13 +1,71 @@ -use std::time::Instant; +use std::{pin::pin, time::Instant}; -use aha::{chat::ChatCompletionParameters, models::lfm2vl::generate::Lfm2VLGenerateModel}; +use aha::{ + chat::ChatCompletionParameters, + models::{GenerateModel, lfm2vl::generate::Lfm2VLGenerateModel}, +}; use anyhow::Result; +use rocket::futures::StreamExt; #[test] fn lfm2vl_generate() -> Result<()> { // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_lfm2vl lfm2vl_generate -r -- --nocapture - + let save_dir = aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; + // let model_path = format!("{}/LiquidAI/LFM2.5-VL-1.6B/", save_dir); + let model_path = format!("{}/LiquidAI/LFM2-VL-1.6B/", save_dir); + let message = r#" + { + "model": "lfm2vl", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image", + "image_url": + { + "url": "file://./assets/img/ocr_test1.png" + } + }, + { + "type": "text", + "text": "图片里面是什么" + } + ] + } + ] + } + "#; + let mes: ChatCompletionParameters = serde_json::from_str(message)?; + let i_start = Instant::now(); + let mut model = Lfm2VLGenerateModel::init(&model_path, None, None)?; + let i_duration = i_start.elapsed(); + println!("Time elapsed in load model is: {:?}", i_duration); + + let i_start = Instant::now(); + let result = model.generate(mes)?; + let i_duration = i_start.elapsed(); + println!("generate: \n {:?}", result); + if let Some(usage) = &result.usage { + let num_token = usage.total_tokens; + let duration_secs = i_duration.as_secs_f64(); + let tps = num_token as f64 / duration_secs; + println!("Tokens per second (TPS): {:.2}", tps); + } + println!("Time elapsed in generate is: {:?}", i_duration); + + Ok(()) +} + +#[tokio::test] +async fn lfm2vl_stream() -> Result<()> { + // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_lfm2vl lfm2vl_stream -r -- --nocapture + // test with cuda+flash-attn: RUST_BACKTRACE=1 cargo test -F cuda,flash-attn qwen3_0_6b_generate -r -- --nocapture + + let save_dir = + aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; + // let model_path = format!("{}/LiquidAI/LFM2-1.2B/", save_dir); let model_path = format!("{}/LiquidAI/LFM2.5-VL-1.6B/", save_dir); let message = r#" { @@ -39,16 +97,13 @@ fn lfm2vl_generate() -> Result<()> { println!("Time elapsed in load model is: {:?}", i_duration); let i_start = Instant::now(); - let result = model.generate(mes)?; + // let result = model.generate(mes)?; + let mut stream = pin!(model.generate_stream(mes)?); let i_duration = i_start.elapsed(); - // println!("generate: \n {:?}", result); - // if let Some(usage) = &result.usage { - // let num_token = usage.total_tokens; - // let duration_secs = i_duration.as_secs_f64(); - // let tps = num_token as f64 / duration_secs; - // println!("Tokens per second (TPS): {:.2}", tps); - // } - // println!("Time elapsed in generate is: {:?}", i_duration); + while let Some(token) = stream.next().await { + println!("generate: \n {:?}", token); + } + println!("Time elapsed in generate is: {:?}", i_duration); Ok(()) -} \ No newline at end of file +} diff --git a/tests/weight_test.rs b/tests/weight_test.rs index cd949b8..f80389d 100644 --- a/tests/weight_test.rs +++ b/tests/weight_test.rs @@ -287,3 +287,26 @@ fn lfm2_weight() -> Result<()> { println!("model_list: {:?}", model_list); Ok(()) } + +#[test] +fn lfm2vl_weight() -> Result<()> { + // cargo test -F cuda --test weight_test lfm2vl_weight -r -- --nocapture + let save_dir = + aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; + // let model_path = format!("{}/LiquidAI/LFM2.5-VL-1.6B/", save_dir); + let model_path = format!("{}/LiquidAI/LFM2-VL-1.6B/", save_dir); + let model_list = find_type_files(&model_path, "safetensors")?; + + let device = Device::Cpu; + for m in &model_list { + let weights = safetensors::load(m, &device)?; + for (key, tensor) in weights.iter() { + // if key.contains("lm_head") { + // println!("=== {} === {:?}", key, tensor.shape()); + // } + println!("=== {} === {:?}", key, tensor.shape()); + } + } + println!("model_list: {:?}", model_list); + Ok(()) +}