tckm.yan/aha
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add LFM2.5-VL-1.6B|LFM2-VL-1.6B

Browse Source
This commit is contained in:
jhqxxx
2026-03-30 00:44:41 +08:00
parent b881dfcd8d
commit ea78da7834
27 changed files with 847 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+4 -8
View File
@@ -44,7 +44,7 @@ dependencies = [
"num",
"rayon",
"realfft",
"reqwest 0.12.28",
"reqwest 0.13.2",
"rocket",
"sentencepiece",
"serde",
@@ -2805,9 +2805,9 @@ dependencies = [
[[package]]
name = "modelscope"
version = "0.1.4"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c0ce1b3f708ede609c998fc460d5dce1b8b9458f99d71715fc8735f8335d8f3"
checksum = "e70d3712581a1c6148246dc7cefd63d2dbb3a40ba70602170adbec34bb7319bf"
dependencies = [
"anyhow",
"clap",
@@ -3810,21 +3810,16 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64 0.22.1",
"bytes",
"encoding_rs",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.4.13",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.8.1",
"hyper-rustls",
"hyper-tls",
"hyper-util",
"js-sys",
"log",
"mime",
"mime_guess",
"native-tls",
"percent-encoding",
@@ -3858,6 +3853,7 @@ dependencies = [
"cookie",
"cookie_store",
"encoding_rs",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.4.13",
Cargo.toml
+2 -2
View File
@@ -16,7 +16,7 @@ serde_json = "1.0.145"
anyhow = "1.0.100"
ffmpeg-next = { version = "8.0.0", optional = true }
image = "0.25.8"
reqwest = { version = "0.12.23", features = ["blocking"] }
reqwest = { version = "0.13", features = ["blocking"] }
base64 = "0.22.1"
num = "0.4.3"
minijinja = "2.12.0"
@@ -28,7 +28,7 @@ rocket = { version = "0.5.1", features = ["serde_json", "json"] }
tokio = "1.47.1"
hound = "3.5.1"
clap = { version = "4.5.51", features = ["derive"] }
modelscope = "0.1.4"
modelscope = "0.1.5"
dirs = "6.0.0"
sysinfo = "0.33"
url = "2.5.7"
README.md
+1 -1
View File
@@ -214,7 +214,7 @@ Apache-2.0 — See [LICENSE](LICENSE) for details.
- All model authors and contributors
## Wechat
![260326 expired](./assets/img/aha_weixingqun.jpg)
![260405 expired](./assets/img/aha_weixingqun.png)
---
<p align="center">
README.zh-CN.md
+3
View File
@@ -218,6 +218,9 @@ Apache-2.0 &mdash; 详见 [LICENSE](LICENSE)
- [Candle](https://github.com/huggingface/candle) - 优秀的 Rust 机器学习框架
- 所有模型作者和贡献者
## Wechat
![260405 expired](./assets/img/aha_weixinqun.png)
---
<p align="center">
assets/img/aha_weixingqun.jpg
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 138 KiB

assets/img/aha_weixinqun.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 152 KiB

src/api/mod.rs
+27 -18
View File
@@ -249,8 +249,10 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str {
WhichModel::MiniCPM4_0_5B => "minicpm4-0.5b",
WhichModel::LFM2_1_2B => "lfm2-1.2b",
WhichModel::LFM2_5_1_2BInstruct => "lfm2.5-1.2b-instruct",
WhichModel::Qwen2_5vl3B => "qwen2.5vl-3b",
WhichModel::Qwen2_5vl7B => "qwen2.5vl-7b",
WhichModel::LFM2_5VL1_6B => "lfm2.5-vl-1.6b",
WhichModel::LFM2VL1_6B => "lfm2-vl-1.6b",
WhichModel::Qwen2_5VL3B => "qwen2.5vl-3b",
WhichModel::Qwen2_5VL7B => "qwen2.5vl-7b",
WhichModel::Qwen3_0_6B => "qwen3-0.6b",
WhichModel::Qwen3_5_0_8B => "qwen3.5-0.8b",
WhichModel::Qwen3_5_2B => "qwen3.5-2b",
@@ -259,10 +261,10 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str {
WhichModel::Qwen3_5Gguf => "qwen3.5-gguf",
WhichModel::Qwen3ASR0_6B => "qwen3asr-0.6b",
WhichModel::Qwen3ASR1_7B => "qwen3asr-1.7b",
WhichModel::Qwen3vl2B => "qwen3vl-2b",
WhichModel::Qwen3vl4B => "qwen3vl-4b",
WhichModel::Qwen3vl8B => "qwen3vl-8b",
WhichModel::Qwen3vl32B => "qwen3vl-32b",
WhichModel::Qwen3VL2B => "qwen3vl-2b",
WhichModel::Qwen3VL4B => "qwen3vl-4b",
WhichModel::Qwen3VL8B => "qwen3vl-8b",
WhichModel::Qwen3VL32B => "qwen3vl-32b",
WhichModel::DeepSeekOCR => "deepseek-ocr",
WhichModel::DeepSeekOCR2 => "deepseek-ocr2",
WhichModel::HunyuanOCR => "hunyuan-ocr",
@@ -281,12 +283,12 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str {
fn which_model_to_owner(which_model: WhichModel) -> &'static str {
match which_model {
WhichModel::MiniCPM4_0_5B => "OpenBMB",
WhichModel::Qwen2_5vl3B | WhichModel::Qwen2_5vl7B => "Qwen",
WhichModel::Qwen2_5VL3B | WhichModel::Qwen2_5VL7B => "Qwen",
WhichModel::Qwen3_0_6B | WhichModel::Qwen3ASR0_6B | WhichModel::Qwen3ASR1_7B => "Qwen",
WhichModel::Qwen3vl2B
| WhichModel::Qwen3vl4B
| WhichModel::Qwen3vl8B
| WhichModel::Qwen3vl32B
WhichModel::Qwen3VL2B
| WhichModel::Qwen3VL4B
| WhichModel::Qwen3VL8B
| WhichModel::Qwen3VL32B
| WhichModel::Qwen3_5Gguf => "Qwen",
WhichModel::Qwen3_5_0_8B
| WhichModel::Qwen3_5_2B
@@ -299,7 +301,10 @@ fn which_model_to_owner(which_model: WhichModel) -> &'static str {
WhichModel::VoxCPM | WhichModel::VoxCPM1_5 => "OpenBMB",
WhichModel::GlmASRNano2512 | WhichModel::GlmOCR => "ZhipuAI",
WhichModel::FunASRNano2512 => "FunAudioLLM",
WhichModel::LFM2_1_2B | WhichModel::LFM2_5_1_2BInstruct => "LiquidAI",
WhichModel::LFM2_1_2B
| WhichModel::LFM2_5_1_2BInstruct
| WhichModel::LFM2_5VL1_6B
| WhichModel::LFM2VL1_6B => "LiquidAI",
}
}
@@ -380,13 +385,13 @@ mod tests {
#[test]
fn test_get_model_type_llm() {
assert_eq!(WhichModel::Qwen3_0_6B.model_type(), "llm");
assert_eq!(WhichModel::Qwen3vl2B.model_type(), "llm");
assert_eq!(WhichModel::Qwen3VL2B.model_type(), "llm");
assert_eq!(WhichModel::MiniCPM4_0_5B.model_type(), "llm");
assert_eq!(WhichModel::Qwen2_5vl3B.model_type(), "llm");
assert_eq!(WhichModel::Qwen2_5vl7B.model_type(), "llm");
assert_eq!(WhichModel::Qwen3vl4B.model_type(), "llm");
assert_eq!(WhichModel::Qwen3vl8B.model_type(), "llm");
assert_eq!(WhichModel::Qwen3vl32B.model_type(), "llm");
assert_eq!(WhichModel::Qwen2_5VL3B.model_type(), "llm");
assert_eq!(WhichModel::Qwen2_5VL7B.model_type(), "llm");
assert_eq!(WhichModel::Qwen3VL4B.model_type(), "llm");
assert_eq!(WhichModel::Qwen3VL8B.model_type(), "llm");
assert_eq!(WhichModel::Qwen3VL32B.model_type(), "llm");
}
#[test]
@@ -407,6 +412,10 @@ mod tests {
#[test]
fn test_get_model_type_image() {
assert_eq!(WhichModel::RMBG2_0.model_type(), "image");
}
#[test]
fn test_get_model_type_tts() {
assert_eq!(WhichModel::VoxCPM.model_type(), "image");
assert_eq!(WhichModel::VoxCPM1_5.model_type(), "image");
}
src/exec/lfm2.rs
+1 -1
View File
@@ -1,4 +1,4 @@
//! MiniCPM4-0.5B exec implementation for CLI `run` subcommand
//! LFM2 exec implementation for CLI `run` subcommand
use std::time::Instant;
src/exec/lfm2vl.rs
+76
View File
@@ -0,0 +1,76 @@
//! LFM2.5VL exec implementation for CLI `run` subcommand
use std::time::Instant;
use anyhow::{Ok, Result};
use crate::exec::ExecModel;
use crate::models::GenerateModel;
use crate::models::lfm2vl::generate::Lfm2VLGenerateModel;
use crate::utils::get_file_path;
pub struct Lfm2VLExec;
impl ExecModel for Lfm2VLExec {
fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
let input_text = &input[0];
let target_text = if input_text.starts_with("file://") {
let path = get_file_path(input_text)?;
std::fs::read_to_string(path)?
} else {
input_text.clone()
};
let url = &input[1];
let input_url = if url.starts_with("http://")
|| url.starts_with("https://")
|| url.starts_with("file://")
{
url.clone()
} else {
format!("file://{}", url)
};
let i_start = Instant::now();
let mut model = Lfm2VLGenerateModel::init(weight_path, None, None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
let message = format!(
r#"{{
"model": "lfm2.5vl",
"messages": [
{{
"role": "user",
"content": [
{{
"type": "image",
"image_url": {{
"url": "{}"
}}
}},
{{
"type": "text",
"text": "{}"
}}
]
}}
]
}}"#,
input_url, target_text
);
let mes = serde_json::from_str(&message)?;
let i_start = Instant::now();
let result = model.generate(mes)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in generate is: {:?}", i_duration);
println!("Result: {:?}", result);
if let Some(out) = output {
std::fs::write(out, format!("{:?}", result))?;
println!("Output saved to: {}", out);
}
Ok(())
}
}
src/exec/mod.rs
+1
View File
@@ -9,6 +9,7 @@ pub mod glm_asr_nano;
pub mod glm_ocr;
pub mod hunyuan_ocr;
pub mod lfm2;
pub mod lfm2vl;
pub mod minicpm4;
pub mod paddleocr_vl;
pub mod qwen2_5vl;
src/exec/qwen2_5vl.rs
+2 -2
View File
@@ -8,9 +8,9 @@ use crate::exec::ExecModel;
use crate::models::{GenerateModel, qwen2_5vl::generate::Qwen2_5VLGenerateModel};
use crate::utils::get_file_path;
pub struct Qwen2_5vlExec;
pub struct Qwen2_5VLExec;
impl ExecModel for Qwen2_5vlExec {
impl ExecModel for Qwen2_5VLExec {
fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
let input_text = &input[0];
let target_text = if input_text.starts_with("file://") {
src/exec/qwen3_asr.rs
+1 -1
View File
@@ -1,4 +1,4 @@
//! Fun-ASR-Nano-2512 exec implementation for CLI `run` subcommand
//! Qwen3ASR exec implementation for CLI `run` subcommand
use std::time::Instant;
src/exec/qwen3vl.rs
+2 -2
View File
@@ -8,9 +8,9 @@ use crate::exec::ExecModel;
use crate::models::{GenerateModel, qwen3vl::generate::Qwen3VLGenerateModel};
use crate::utils::get_file_path;
pub struct Qwen3vlExec;
pub struct Qwen3VLExec;
impl ExecModel for Qwen3vlExec {
impl ExecModel for Qwen3VLExec {
fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
let input_text = &input[0];
let target_text = if input_text.starts_with("file://") {
src/main.rs
+32 -24
View File
@@ -246,8 +246,8 @@ fn run_list(args: ListArgs) -> anyhow::Result<()> {
WhichModel::MiniCPM4_0_5B,
WhichModel::LFM2_1_2B,
WhichModel::LFM2_5_1_2BInstruct,
WhichModel::Qwen2_5vl3B,
WhichModel::Qwen2_5vl7B,
WhichModel::Qwen2_5VL3B,
WhichModel::Qwen2_5VL7B,
WhichModel::Qwen3_0_6B,
WhichModel::Qwen3_5_0_8B,
WhichModel::Qwen3_5_2B,
@@ -255,10 +255,10 @@ fn run_list(args: ListArgs) -> anyhow::Result<()> {
WhichModel::Qwen3_5_9B,
WhichModel::Qwen3ASR0_6B,
WhichModel::Qwen3ASR1_7B,
WhichModel::Qwen3vl2B,
WhichModel::Qwen3vl4B,
WhichModel::Qwen3vl8B,
WhichModel::Qwen3vl32B,
WhichModel::Qwen3VL2B,
WhichModel::Qwen3VL4B,
WhichModel::Qwen3VL8B,
WhichModel::Qwen3VL32B,
WhichModel::DeepSeekOCR,
WhichModel::DeepSeekOCR2,
WhichModel::HunyuanOCR,
@@ -471,13 +471,21 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
use aha::exec::lfm2::Lfm2Exec;
Lfm2Exec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen2_5vl3B => {
use aha::exec::qwen2_5vl::Qwen2_5vlExec;
Qwen2_5vlExec::run(&input, output.as_deref(), &weight_path)?;
WhichModel::LFM2_5VL1_6B => {
use aha::exec::lfm2vl::Lfm2VLExec;
Lfm2VLExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen2_5vl7B => {
use aha::exec::qwen2_5vl::Qwen2_5vlExec;
Qwen2_5vlExec::run(&input, output.as_deref(), &weight_path)?;
WhichModel::LFM2VL1_6B => {
use aha::exec::lfm2vl::Lfm2VLExec;
Lfm2VLExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen2_5VL3B => {
use aha::exec::qwen2_5vl::Qwen2_5VLExec;
Qwen2_5VLExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen2_5VL7B => {
use aha::exec::qwen2_5vl::Qwen2_5VLExec;
Qwen2_5VLExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen3_0_6B => {
use aha::exec::qwen3::Qwen3Exec;
@@ -511,21 +519,21 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
use aha::exec::qwen3_asr::Qwen3ASRExec;
Qwen3ASRExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen3vl2B => {
use aha::exec::qwen3vl::Qwen3vlExec;
Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?;
WhichModel::Qwen3VL2B => {
use aha::exec::qwen3vl::Qwen3VLExec;
Qwen3VLExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen3vl4B => {
use aha::exec::qwen3vl::Qwen3vlExec;
Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?;
WhichModel::Qwen3VL4B => {
use aha::exec::qwen3vl::Qwen3VLExec;
Qwen3VLExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen3vl8B => {
use aha::exec::qwen3vl::Qwen3vlExec;
Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?;
WhichModel::Qwen3VL8B => {
use aha::exec::qwen3vl::Qwen3VLExec;
Qwen3VLExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen3vl32B => {
use aha::exec::qwen3vl::Qwen3vlExec;
Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?;
WhichModel::Qwen3VL32B => {
use aha::exec::qwen3vl::Qwen3VLExec;
Qwen3VLExec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::DeepSeekOCR => {
use aha::exec::deepseek_ocr::DeepSeekORExec;
src/models/lfm2/model.rs
+1 -1
View File
@@ -197,7 +197,7 @@ impl Lfm2DecoderLayer {
}
pub struct Lfm2Decoder {
embed_tokens: Embedding,
pub embed_tokens: Embedding,
layers: Vec<Lfm2DecoderLayer>,
// rotary_emb: RoPE,
pos_emb: RoPE,
src/models/lfm2vl/config.rs
+3 -6
View File
@@ -2,7 +2,6 @@ use candle_nn::Activation;
use crate::models::lfm2::config::Lfm2Config;
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
pub struct Lfm2VLConfig {
pub do_image_splitting: bool,
@@ -19,7 +18,7 @@ pub struct Lfm2VLConfig {
pub projector_bias: bool,
pub projector_hidden_act: Activation,
pub projector_hidden_size: usize,
pub projector_use_layernorm: bool,
pub projector_use_layernorm: Option<bool>,
pub text_config: Lfm2Config,
pub tile_size: usize,
pub use_image_special_tokens: bool,
@@ -27,7 +26,6 @@ pub struct Lfm2VLConfig {
pub vision_config: Lfm2VLVisionConfig,
}
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
pub struct Lfm2VLVisionConfig {
pub attention_dropout: f64,
@@ -37,8 +35,8 @@ pub struct Lfm2VLVisionConfig {
pub intermediate_size: usize,
pub layer_norm_eps: f64,
pub model_type: String,
pub num_attention_heads: u32,
pub num_channels: u32,
pub num_attention_heads: usize,
pub num_channels: usize,
pub num_hidden_layers: usize,
pub num_patches: usize,
pub patch_size: usize,
@@ -79,4 +77,3 @@ pub struct Size {
pub struct Lfm2ProcessorConfig {
pub image_processor: Lfm2ImageConfig,
}
src/models/lfm2vl/generate.rs
+137 -15
View File
@@ -1,22 +1,28 @@
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
use aha_openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatCompletionResponse};
use anyhow::Result;
use candle_core::{DType, Device};
use candle_core::{DType, Device, Tensor};
use candle_nn::VarBuilder;
use crate::{
chat_template::ChatTemplate,
models::{
GenerateModel,
lfm2::config::Lfm2GenerateConfig,
lfm2vl::{config::Lfm2VLConfig, processor::Lfm2VLProcessor},
lfm2vl::{config::Lfm2VLConfig, model::Lfm2VLModel, processor::Lfm2VLProcessor},
},
tokenizer::TokenizerModel,
utils::{find_type_files, get_device, get_dtype, get_logit_processor},
utils::{
build_completion_chunk_response, build_completion_response, find_type_files, get_device,
get_dtype, get_logit_processor,
},
};
use rocket::async_stream::stream;
pub struct Lfm2VLGenerateModel<'a> {
chat_template: ChatTemplate<'a>,
tokenizer: TokenizerModel,
device: Device,
// model: Lfm2VLModel,
model: Lfm2VLModel,
processor: Lfm2VLProcessor,
eos_token_id: u32,
model_name: String,
@@ -33,8 +39,8 @@ impl<'a> Lfm2VLGenerateModel<'a> {
let model_path = find_type_files(path, "safetensors")?;
let dtype = get_dtype(dtype, &cfg.dtype);
// let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_path, dtype, &device)? };
// let model = Lfm2Model::new(vb, &cfg)?;
let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_path, dtype, &device)? };
let model = Lfm2VLModel::new(vb, &cfg)?;
let processor = Lfm2VLProcessor::new(path, dtype, &device)?;
let eos_token_id = gen_cfg.eos_token_id;
let model_name = std::path::Path::new(path)
@@ -46,14 +52,16 @@ impl<'a> Lfm2VLGenerateModel<'a> {
chat_template,
tokenizer,
device,
// model,
model,
processor,
eos_token_id,
model_name,
})
}
}
pub fn generate(&mut self, mes: ChatCompletionParameters) -> Result<()> {
impl<'a> GenerateModel for Lfm2VLGenerateModel<'a> {
fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
let mes_render = self.chat_template.apply_chat_template(&mes)?;
let mut logits = get_logit_processor(
mes.temperature,
@@ -63,11 +71,125 @@ impl<'a> Lfm2VLGenerateModel<'a> {
);
let (pixel_values, pixel_attention_mask, spatial_shapes, text) =
self.processor.process_info(&mes, &mes_render)?;
let input_ids = self.tokenizer.text_encode(text, &self.device)?;
println!("pixel_values: {}", pixel_values);
println!("pixel_attention_mask: {}", pixel_attention_mask);
println!("spatial_shapes: {}", spatial_shapes);
println!("input_ids: {}", input_ids);
Ok(())
let mut input_ids = self.tokenizer.text_encode(text, &self.device)?;
let mut seq_len = input_ids.dim(1)?;
let prompt_tokens = seq_len as u32;
let mut seqlen_offset = 0;
let mut generate = vec![];
let sample_len = mes.max_tokens.unwrap_or(1024);
let mut pixel_values = Some(pixel_values);
let mut pixel_attention_mask = Some(pixel_attention_mask);
let mut spatial_shapes = Some(spatial_shapes);
for _ in 0..sample_len {
let logit = self.model.forward(
&input_ids,
pixel_values.as_ref(),
pixel_attention_mask.as_ref(),
spatial_shapes.as_ref(),
seqlen_offset,
)?;
let logit = logit.squeeze(0)?.squeeze(0)?;
let next_token = logits.sample(&logit)?;
generate.push(next_token);
if next_token == self.eos_token_id {
break;
}
input_ids = Tensor::new(vec![next_token], &self.device)?.unsqueeze(0)?;
seqlen_offset += seq_len;
seq_len = 1;
pixel_values = None;
pixel_attention_mask = None;
spatial_shapes = None;
}
self.model.clear_cache();
let completion_tokens = generate.len() as u32;
let decode = self.tokenizer.token_decode(generate)?;
let mes = build_completion_response(
decode,
&self.model_name,
Some(completion_tokens),
Some(prompt_tokens),
);
Ok(mes)
}
fn generate_stream(
&mut self,
mes: ChatCompletionParameters,
) -> Result<
Box<
dyn rocket::futures::Stream<
Item = Result<
aha_openai_dive::v1::resources::chat::ChatCompletionChunkResponse,
anyhow::Error,
>,
> + Send
+ Unpin
+ '_,
>,
> {
let mes_render = self.chat_template.apply_chat_template(&mes)?;
let mut logits = get_logit_processor(
mes.temperature,
mes.top_p,
None,
mes.seed.unwrap_or(34562) as u64,
);
let (pixel_values, pixel_attention_mask, spatial_shapes, text) =
self.processor.process_info(&mes, &mes_render)?;
let mut input_ids = self.tokenizer.text_encode(text, &self.device)?;
let mut seq_len = input_ids.dim(1)?;
let mut seqlen_offset = 0;
let sample_len = mes.max_tokens.unwrap_or(1024);
let mut pixel_values = Some(pixel_values);
let mut pixel_attention_mask = Some(pixel_attention_mask);
let mut spatial_shapes = Some(spatial_shapes);
let stream = stream! {
let mut err_tokens = vec![];
for _ in 0..sample_len {
let logit = self.model.forward(
&input_ids,
pixel_values.as_ref(),
pixel_attention_mask.as_ref(),
spatial_shapes.as_ref(),
seqlen_offset,
)?;
let logit = logit.squeeze(0)?.squeeze(0)?;
let next_token = logits.sample(&logit)?;
let mut decode_ids = vec![];
if !err_tokens.is_empty() {
decode_ids.extend_from_slice(&err_tokens);
}
decode_ids.push(next_token);
let decode = self.tokenizer.token_decode(decode_ids)?;
if decode.contains("�") {
err_tokens.push(next_token);
if err_tokens.len() > 3 {
err_tokens.clear();
}
input_ids = Tensor::new(vec![next_token], &self.device)?.unsqueeze(0)?;
seqlen_offset += seq_len;
seq_len = 1;
pixel_values = None;
pixel_attention_mask = None;
spatial_shapes = None;
continue;
}
err_tokens.clear();
let chunk = build_completion_chunk_response(decode, &self.model_name, None, None);
yield Ok(chunk);
if next_token == self.eos_token_id {
break;
}
input_ids = Tensor::new(vec![next_token], &self.device)?.unsqueeze(0)?;
seqlen_offset += seq_len;
seq_len = 1;
pixel_values = None;
pixel_attention_mask = None;
spatial_shapes = None;
}
self.model.clear_cache();
};
Ok(Box::new(Box::pin(stream)))
}
}
src/models/lfm2vl/mod.rs
+1 -1
View File
@@ -1,4 +1,4 @@
pub mod config;
pub mod generate;
pub mod model;
pub mod processor;
pub mod processor;
src/models/lfm2vl/model.rs
+325
View File
@@ -0,0 +1,325 @@
use crate::{
models::{
common::{NaiveAttnTwoLinearMLPBlock, get_layer_norm},
lfm2::model::Lfm2Decoder,
lfm2vl::config::{Lfm2VLConfig, Lfm2VLVisionConfig},
},
utils::{
interpolate::interpolate_bilinear,
tensor_utils::{get_equal_mask, masked_scatter_dim0, prepare_mask},
},
};
use anyhow::{Result, anyhow};
use candle_core::{D, IndexOp, Tensor};
use candle_nn::{Activation, LayerNorm, Linear, Module, VarBuilder, embedding, linear_b};
use num::integer::Roots;
pub struct Siglip2VisionEmbeddings {
// embed_dim: usize,
// patch_size: usize,
patch_embedding: Linear,
// position_embedding_size: usize,
// position_embedding: Embedding,
postitional_embeddings: Tensor,
}
impl Siglip2VisionEmbeddings {
pub fn new(vb: VarBuilder, cfg: &Lfm2VLVisionConfig) -> Result<Self> {
let embed_dim = cfg.hidden_size;
let patch_size = cfg.patch_size;
let patch_embedding = linear_b(
cfg.num_channels * patch_size * patch_size,
embed_dim,
true,
vb.pp("patch_embedding"),
)?;
let position_embedding_size = cfg.num_patches.sqrt();
let position_embedding =
embedding(cfg.num_patches, embed_dim, vb.pp("position_embedding"))?;
// (num_patches, embed_dim) -> (h, w, embed_dim)
// ->(embed_dim, h, w) -> (1, embe_dim, h, w)
let postitional_embeddings = position_embedding
.embeddings()
.reshape((position_embedding_size, position_embedding_size, ()))?
.permute((2, 0, 1))?
.unsqueeze(0)?;
Ok(Self {
// embed_dim,
// patch_size,
patch_embedding,
// position_embedding_size,
// position_embedding,
postitional_embeddings,
})
}
fn resize_positional_embeddings(
&self,
spatial_shapes: &Tensor,
max_length: usize,
) -> Result<Tensor> {
let mut result_pos_embeddings = vec![];
let bs = spatial_shapes.dim(0)?;
for i in 0..bs {
let shape_i = spatial_shapes.i(i)?.to_vec1::<u32>()?;
let height = *shape_i.first().unwrap_or(&32) as usize;
let width = *shape_i.get(1).unwrap_or(&32) as usize;
if height == 0 || width == 0 || height * width > max_length {
return Err(anyhow!("img height or width illigal"));
}
// (1, dim, height, width) -> (height*width, dim)
let resize_embeddings = interpolate_bilinear(
&self.postitional_embeddings,
(height, width),
Some(false),
Some(true),
)?
.reshape(((), height * width))?
.transpose(0, 1)?;
let resize_embeddings = if height * width < max_length {
let pad = max_length - height * width;
let pad_embedding = resize_embeddings.i(0)?.unsqueeze(0)?.repeat((pad, 1))?;
Tensor::cat(&[&resize_embeddings, &pad_embedding], 0)?
} else {
resize_embeddings
};
result_pos_embeddings.push(resize_embeddings);
}
let result_pos_embeddings = Tensor::stack(&result_pos_embeddings, 0)?;
Ok(result_pos_embeddings)
}
pub fn forward(&self, pixel_values: &Tensor, spatial_shapes: &Tensor) -> Result<Tensor> {
let patch_embeds = self.patch_embedding.forward(pixel_values)?;
let max_length = pixel_values.dim(1)?;
let resize_pos_embeddings =
self.resize_positional_embeddings(spatial_shapes, max_length)?;
let embedding = patch_embeds.add(&resize_pos_embeddings)?;
Ok(embedding)
}
}
pub struct Siglip2Encoder {
layers: Vec<NaiveAttnTwoLinearMLPBlock>,
}
impl Siglip2Encoder {
pub fn new(vb: VarBuilder, cfg: &Lfm2VLVisionConfig) -> Result<Self> {
let vb_layers = vb.pp("layers");
let mut layers = vec![];
for i in 0..cfg.num_hidden_layers {
let layer = NaiveAttnTwoLinearMLPBlock::new(
vb_layers.pp(i),
cfg.hidden_size,
cfg.num_attention_heads,
None,
None,
true,
"self_attn",
Some("out_proj"),
cfg.intermediate_size,
cfg.hidden_act,
true,
"mlp",
"fc1",
"fc2",
cfg.layer_norm_eps,
"layer_norm1",
"layer_norm2",
)?;
layers.push(layer);
}
Ok(Self { layers })
}
pub fn forward(&self, xs: &Tensor, attention_mask: Option<&Tensor>) -> Result<Tensor> {
let mut xs = xs.clone();
for layer in &self.layers {
xs = layer.forward(&xs, None, None, attention_mask, false)?;
}
Ok(xs)
}
}
pub struct Siglip2VisionModel {
embeddings: Siglip2VisionEmbeddings,
encoder: Siglip2Encoder,
post_layernorm: LayerNorm,
}
impl Siglip2VisionModel {
pub fn new(vb: VarBuilder, cfg: &Lfm2VLVisionConfig) -> Result<Self> {
let vb = vb.pp("vision_model");
let embeddings = Siglip2VisionEmbeddings::new(vb.pp("embeddings"), cfg)?;
let encoder = Siglip2Encoder::new(vb.pp("encoder"), cfg)?;
let post_layernorm = get_layer_norm(
vb.pp("post_layernorm"),
cfg.layer_norm_eps,
cfg.hidden_size,
true,
)?;
Ok(Self {
embeddings,
encoder,
post_layernorm,
})
}
pub fn forward(
&self,
pixel_values: &Tensor,
attention_mask: &Tensor,
spatial_shapes: &Tensor,
) -> Result<Tensor> {
let xs = self.embeddings.forward(pixel_values, spatial_shapes)?;
let mask = prepare_mask(attention_mask)?.to_dtype(xs.dtype())?;
let xs = self.encoder.forward(&xs, Some(&mask))?;
let xs = self.post_layernorm.forward(&xs)?;
Ok(xs)
}
}
pub struct Lfm2VlMultiModalProjector {
factor: usize,
layer_norm: Option<LayerNorm>,
linear_1: Linear,
act: Activation,
linear_2: Linear,
}
impl Lfm2VlMultiModalProjector {
pub fn new(vb: VarBuilder, cfg: &Lfm2VLConfig) -> Result<Self> {
let in_channels = cfg.vision_config.hidden_size * (cfg.downsample_factor).pow(2);
let factor = cfg.downsample_factor;
let layer_norm = if let Some(flag) = cfg.projector_use_layernorm
&& !flag
{
None
} else {
let layer_norm = get_layer_norm(
vb.pp("layer_norm"),
cfg.vision_config.layer_norm_eps,
in_channels,
true,
)?;
Some(layer_norm)
};
let linear_1 = linear_b(
in_channels,
cfg.projector_hidden_size,
cfg.projector_bias,
vb.pp("linear_1"),
)?;
let act = cfg.projector_hidden_act;
let linear_2 = linear_b(
cfg.projector_hidden_size,
cfg.text_config.hidden_size,
cfg.projector_bias,
vb.pp("linear_2"),
)?;
Ok(Self {
factor,
layer_norm,
linear_1,
act,
linear_2,
})
}
pub fn forward(&self, xs: &Tensor) -> Result<Tensor> {
let (bs, w, h, c) = xs.dims4()?;
let xs = xs.reshape((bs, w, h / self.factor, c * self.factor))?;
let xs = xs.permute((0, 2, 1, 3))?;
let xs = xs.reshape((
bs,
h / self.factor,
w / self.factor,
c * self.factor * self.factor,
))?;
let mut xs = xs.permute((0, 2, 1, 3))?.contiguous()?;
if let Some(norm) = &self.layer_norm {
xs = norm.forward(&xs)?;
}
xs = self.linear_1.forward(&xs)?.apply(&self.act)?;
xs = self.linear_2.forward(&xs)?;
Ok(xs)
}
}
pub struct Lfm2VLModel {
vision_tower: Siglip2VisionModel,
multi_modal_projector: Lfm2VlMultiModalProjector,
language_model: Lfm2Decoder,
lm_head: Linear,
img_id: u32,
}
impl Lfm2VLModel {
pub fn new(vb: VarBuilder, cfg: &Lfm2VLConfig) -> Result<Self> {
let vb = vb.pp("model");
let vision_tower = Siglip2VisionModel::new(vb.pp("vision_tower"), &cfg.vision_config)?;
let multi_modal_projector =
Lfm2VlMultiModalProjector::new(vb.pp("multi_modal_projector"), cfg)?;
let language_model = Lfm2Decoder::new(vb.pp("language_model"), &cfg.text_config)?;
let lm_head = Linear::new(language_model.embed_tokens.embeddings().clone(), None);
Ok(Self {
vision_tower,
multi_modal_projector,
language_model,
lm_head,
img_id: cfg.image_token_id,
})
}
pub fn forward(
&mut self,
input_ids: &Tensor,
pixel_values: Option<&Tensor>,
pixel_attention_mask: Option<&Tensor>,
spatial_shapes: Option<&Tensor>,
seqlen_offset: usize,
) -> Result<Tensor> {
let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?;
if let Some(pixel) = pixel_values
&& let Some(mask) = pixel_attention_mask
&& let Some(shapes) = spatial_shapes
{
let image_embeds = self.vision_tower.forward(pixel, mask, shapes)?;
println!("image_embeds: {}", image_embeds);
let bs = image_embeds.dim(0)?;
let img_featrure_length = mask.sum(1)?.to_vec1::<u32>()?;
let mut image_features = vec![];
for img_idx in 0..bs {
let feature = image_embeds.i(img_idx)?;
let feature = feature.narrow(0, 0, img_featrure_length[img_idx] as usize)?;
let shape = shapes.i(img_idx)?.to_vec1::<u32>()?;
let h = shape[0];
let w = shape[1];
let feature = feature
.reshape((1, h as usize, w as usize, ()))?
.contiguous()?;
let img_embedding = self.multi_modal_projector.forward(&feature)?;
let dim = img_embedding.dim(D::Minus1)?;
let img_embedding = img_embedding.reshape(((), dim))?;
image_features.push(img_embedding);
}
let image_embeds = Tensor::cat(&image_features, 0)?;
println!("image_embeds: {}", image_embeds);
let image_mask = get_equal_mask(input_ids, self.img_id)?;
inputs_embeds = masked_scatter_dim0(&inputs_embeds, &image_embeds, &image_mask)?;
}
let output = self
.language_model
.forward(input_ids, Some(&inputs_embeds), seqlen_offset)?;
let seq_len = output.dim(1)?;
let last = output.narrow(1, seq_len - 1, 1)?;
let logits = self.lm_head.forward(&last)?;
Ok(logits)
}
pub fn clear_cache(&mut self) {
self.language_model.clear_cache();
}
}
src/models/lfm2vl/processor.rs
+14 -8
View File
@@ -33,17 +33,24 @@ pub struct Lfm2VLProcessor {
image_thumbnail_token: String,
}
#[allow(clippy::type_complexity)]
impl Lfm2VLProcessor {
pub fn new(path: &str, dtype: DType, device: &Device) -> Result<Self> {
let path = path.to_string();
assert!(
std::path::Path::new(&path).exists(),
std::path::Path::new(path).exists(),
"model path file not exists"
);
let processor_cfg_path = path + "/processor_config.json";
let processor_cfg: Lfm2ProcessorConfig =
serde_json::from_slice(&std::fs::read(processor_cfg_path)?)?;
let image_config = processor_cfg.image_processor;
let processor_cfg_path = path.to_string() + "/processor_config.json";
let processor_cfg =
serde_json::from_slice::<Lfm2ProcessorConfig>(&std::fs::read(processor_cfg_path)?);
let image_config = match processor_cfg {
Ok(cfg) => cfg.image_processor,
Err(_) => {
let processor_cfg_path = path.to_string() + "/preprocessor_config.json";
serde_json::from_slice::<Lfm2ImageConfig>(&std::fs::read(processor_cfg_path)?)?
}
};
// 256
let max_thumbnail_image_patches =
image_config.max_image_tokens * image_config.downsample_factor.pow(2);
@@ -160,7 +167,7 @@ impl Lfm2VLProcessor {
let (new_height, new_width) = img_smart_resize(
height,
width,
self.total_factor as u32,
self.total_factor,
self.smart_resize_min_pixels as u32,
self.smart_resize_max_pixels as u32,
)?;
@@ -331,7 +338,6 @@ impl Lfm2VLProcessor {
) = self.process_imgs(imgs)?;
let text =
self.expand_text_with_placeholders(text, num_cols_list, num_rows_list, image_size_list);
Ok((pixel_values, pixel_attention_mask, spatial_shapes, text))
}
}
src/models/mod.rs
+78 -58
View File
@@ -33,70 +33,74 @@ use crate::models::{
fun_asr_nano::generate::FunAsrNanoGenerateModel,
glm_asr_nano::generate::GlmAsrNanoGenerateModel, glm_ocr::generate::GlmOcrGenerateModel,
hunyuan_ocr::generate::HunyuanOCRGenerateModel, lfm2::generate::Lfm2GenerateModel,
minicpm4::generate::MiniCPMGenerateModel, paddleocr_vl::generate::PaddleOCRVLGenerateModel,
qwen2_5vl::generate::Qwen2_5VLGenerateModel, qwen3::generate::Qwen3GenerateModel,
qwen3_5::generate::Qwen3_5GenerateModel, qwen3_asr::generate::Qwen3AsrGenerateModel,
qwen3vl::generate::Qwen3VLGenerateModel, rmbg2_0::generate::RMBG2_0Model,
voxcpm::generate::VoxCPMGenerate,
lfm2vl::generate::Lfm2VLGenerateModel, minicpm4::generate::MiniCPMGenerateModel,
paddleocr_vl::generate::PaddleOCRVLGenerateModel, qwen2_5vl::generate::Qwen2_5VLGenerateModel,
qwen3::generate::Qwen3GenerateModel, qwen3_5::generate::Qwen3_5GenerateModel,
qwen3_asr::generate::Qwen3AsrGenerateModel, qwen3vl::generate::Qwen3VLGenerateModel,
rmbg2_0::generate::RMBG2_0Model, voxcpm::generate::VoxCPMGenerate,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
pub enum WhichModel {
#[value(name = "lfm2-1.2b", hide = true)]
#[value(name = "lfm2-1.2b")]
LFM2_1_2B,
#[value(name = "lfm2.5-1.2b-instruct", hide = true)]
#[value(name = "lfm2.5-1.2b-instruct")]
LFM2_5_1_2BInstruct,
#[value(name = "minicpm4-0.5b", hide = true)]
#[value(name = "lfm2.5-vl-1.6b")]
LFM2_5VL1_6B,
#[value(name = "lfm2-vl-1.6b")]
LFM2VL1_6B,
#[value(name = "minicpm4-0.5b")]
MiniCPM4_0_5B,
#[value(name = "qwen2.5vl-3b", hide = true)]
Qwen2_5vl3B,
#[value(name = "qwen2.5vl-7b", hide = true)]
Qwen2_5vl7B,
#[value(name = "qwen3-0.6b", hide = true)]
#[value(name = "qwen2.5vl-3b")]
Qwen2_5VL3B,
#[value(name = "qwen2.5vl-7b")]
Qwen2_5VL7B,
#[value(name = "qwen3-0.6b")]
Qwen3_0_6B,
#[value(name = "qwen3.5-0.8b", hide = true)]
#[value(name = "qwen3.5-0.8b")]
Qwen3_5_0_8B,
#[value(name = "qwen3.5-2b", hide = true)]
#[value(name = "qwen3.5-2b")]
Qwen3_5_2B,
#[value(name = "qwen3.5-4b", hide = true)]
#[value(name = "qwen3.5-4b")]
Qwen3_5_4B,
#[value(name = "qwen3.5-9b", hide = true)]
#[value(name = "qwen3.5-9b")]
Qwen3_5_9B,
#[value(name = "qwen3.5-gguf", hide = true)]
#[value(name = "qwen3.5-gguf")]
Qwen3_5Gguf,
#[value(name = "qwen3asr-0.6b", hide = true)]
#[value(name = "qwen3asr-0.6b")]
Qwen3ASR0_6B,
#[value(name = "qwen3asr-1.7b", hide = true)]
#[value(name = "qwen3asr-1.7b")]
Qwen3ASR1_7B,
#[value(name = "qwen3vl-2b", hide = true)]
Qwen3vl2B,
#[value(name = "qwen3vl-4b", hide = true)]
Qwen3vl4B,
#[value(name = "qwen3vl-8b", hide = true)]
Qwen3vl8B,
#[value(name = "qwen3vl-32b", hide = true)]
Qwen3vl32B,
#[value(name = "deepseek-ocr", hide = true)]
#[value(name = "qwen3vl-2b")]
Qwen3VL2B,
#[value(name = "qwen3vl-4b")]
Qwen3VL4B,
#[value(name = "qwen3vl-8b")]
Qwen3VL8B,
#[value(name = "qwen3vl-32b")]
Qwen3VL32B,
#[value(name = "deepseek-ocr")]
DeepSeekOCR,
#[value(name = "deepseek-ocr2", hide = true)]
#[value(name = "deepseek-ocr2")]
DeepSeekOCR2,
#[value(name = "hunyuan-ocr", hide = true)]
#[value(name = "hunyuan-ocr")]
HunyuanOCR,
#[value(name = "paddleocr-vl", hide = true)]
#[value(name = "paddleocr-vl")]
PaddleOCRVL,
#[value(name = "paddleocr-vl1.5", hide = true)]
#[value(name = "paddleocr-vl1.5")]
PaddleOCRVL1_5,
#[value(name = "rmbg2.0")]
RMBG2_0,
#[value(name = "voxcpm", hide = true)]
#[value(name = "voxcpm")]
VoxCPM,
#[value(name = "voxcpm1.5", hide = true)]
#[value(name = "voxcpm1.5")]
VoxCPM1_5,
#[value(name = "glm-asr-nano-2512", hide = true)]
#[value(name = "glm-asr-nano-2512")]
GlmASRNano2512,
#[value(name = "fun-asr-nano-2512", hide = true)]
#[value(name = "fun-asr-nano-2512")]
FunASRNano2512,
#[value(name = "glm-ocr", hide = true)]
#[value(name = "glm-ocr")]
GlmOCR,
}
@@ -106,9 +110,11 @@ impl WhichModel {
match self {
WhichModel::LFM2_1_2B => "LiquidAI/LFM2-1.2B",
WhichModel::LFM2_5_1_2BInstruct => "LiquidAI/LFM2.5-1.2B-Instruct",
WhichModel::LFM2_5VL1_6B => "LiquidAI/LFM2.5-VL-1.6B",
WhichModel::LFM2VL1_6B => "LiquidAI/LFM2-VL-1.6B",
WhichModel::MiniCPM4_0_5B => "OpenBMB/MiniCPM4-0.5B",
WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct",
WhichModel::Qwen2_5vl7B => "Qwen/Qwen2.5-VL-7B-Instruct",
WhichModel::Qwen2_5VL3B => "Qwen/Qwen2.5-VL-3B-Instruct",
WhichModel::Qwen2_5VL7B => "Qwen/Qwen2.5-VL-7B-Instruct",
WhichModel::Qwen3_0_6B => "Qwen/Qwen3-0.6B",
WhichModel::Qwen3_5_0_8B => "Qwen/Qwen3.5-0.8B",
WhichModel::Qwen3_5_2B => "Qwen/Qwen3.5-2B",
@@ -117,10 +123,10 @@ impl WhichModel {
WhichModel::Qwen3_5Gguf => "GGUF",
WhichModel::Qwen3ASR0_6B => "Qwen/Qwen3-ASR-0.6B",
WhichModel::Qwen3ASR1_7B => "Qwen/Qwen3-ASR-1.7B",
WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct",
WhichModel::Qwen3vl4B => "Qwen/Qwen3-VL-4B-Instruct",
WhichModel::Qwen3vl8B => "Qwen/Qwen3-VL-8B-Instruct",
WhichModel::Qwen3vl32B => "Qwen/Qwen3-VL-32B-Instruct",
WhichModel::Qwen3VL2B => "Qwen/Qwen3-VL-2B-Instruct",
WhichModel::Qwen3VL4B => "Qwen/Qwen3-VL-4B-Instruct",
WhichModel::Qwen3VL8B => "Qwen/Qwen3-VL-8B-Instruct",
WhichModel::Qwen3VL32B => "Qwen/Qwen3-VL-32B-Instruct",
WhichModel::DeepSeekOCR => "deepseek-ai/DeepSeek-OCR",
WhichModel::DeepSeekOCR2 => "deepseek-ai/DeepSeek-OCR-2",
WhichModel::HunyuanOCR => "Tencent-Hunyuan/HunyuanOCR",
@@ -143,17 +149,19 @@ impl WhichModel {
| WhichModel::Qwen3_0_6B
| WhichModel::LFM2_1_2B
| WhichModel::LFM2_5_1_2BInstruct => "llm",
WhichModel::Qwen2_5vl3B
| WhichModel::Qwen2_5vl7B
| WhichModel::Qwen3vl2B
| WhichModel::Qwen3vl4B
| WhichModel::Qwen3vl8B
| WhichModel::Qwen3vl32B
WhichModel::Qwen2_5VL3B
| WhichModel::Qwen2_5VL7B
| WhichModel::Qwen3VL2B
| WhichModel::Qwen3VL4B
| WhichModel::Qwen3VL8B
| WhichModel::Qwen3VL32B
| WhichModel::Qwen3_5_0_8B
| WhichModel::Qwen3_5_2B
| WhichModel::Qwen3_5_4B
| WhichModel::Qwen3_5_9B
| WhichModel::Qwen3_5Gguf => "vlm",
| WhichModel::Qwen3_5Gguf
| WhichModel::LFM2_5VL1_6B
| WhichModel::LFM2VL1_6B => "vlm",
// OCR models
WhichModel::DeepSeekOCR
| WhichModel::DeepSeekOCR2
@@ -167,7 +175,8 @@ impl WhichModel {
| WhichModel::GlmASRNano2512
| WhichModel::FunASRNano2512 => "asr",
// Image models
WhichModel::RMBG2_0 | WhichModel::VoxCPM | WhichModel::VoxCPM1_5 => "image",
WhichModel::RMBG2_0 => "image",
WhichModel::VoxCPM | WhichModel::VoxCPM1_5 => "tts",
}
}
}
@@ -190,6 +199,7 @@ pub trait GenerateModel {
pub enum ModelInstance<'a> {
MiniCPM4(MiniCPMGenerateModel<'a>),
Lfm2(Lfm2GenerateModel<'a>),
Lfm2VL(Lfm2VLGenerateModel<'a>),
Qwen2_5VL(Qwen2_5VLGenerateModel<'a>),
Qwen3(Qwen3GenerateModel<'a>),
Qwen3_5(Qwen3_5GenerateModel<'a>),
@@ -210,6 +220,7 @@ impl<'a> GenerateModel for ModelInstance<'a> {
match self {
ModelInstance::MiniCPM4(model) => model.generate(mes),
ModelInstance::Lfm2(model) => model.generate(mes),
ModelInstance::Lfm2VL(model) => model.generate(mes),
ModelInstance::Qwen2_5VL(model) => model.generate(mes),
ModelInstance::Qwen3(model) => model.generate(mes),
ModelInstance::Qwen3_5(model) => model.generate(mes),
@@ -240,6 +251,7 @@ impl<'a> GenerateModel for ModelInstance<'a> {
match self {
ModelInstance::MiniCPM4(model) => model.generate_stream(mes),
ModelInstance::Lfm2(model) => model.generate_stream(mes),
ModelInstance::Lfm2VL(model) => model.generate_stream(mes),
ModelInstance::Qwen2_5VL(model) => model.generate_stream(mes),
ModelInstance::Qwen3(model) => model.generate_stream(mes),
ModelInstance::Qwen3_5(model) => model.generate_stream(mes),
@@ -276,11 +288,19 @@ pub fn load_model<'a>(
let model = Lfm2GenerateModel::init(path, None, None)?;
ModelInstance::Lfm2(model)
}
WhichModel::Qwen2_5vl3B => {
WhichModel::LFM2_5VL1_6B => {
let model = Lfm2VLGenerateModel::init(path, None, None)?;
ModelInstance::Lfm2VL(model)
}
WhichModel::LFM2VL1_6B => {
let model = Lfm2VLGenerateModel::init(path, None, None)?;
ModelInstance::Lfm2VL(model)
}
WhichModel::Qwen2_5VL3B => {
let model = Qwen2_5VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen2_5VL(model)
}
WhichModel::Qwen2_5vl7B => {
WhichModel::Qwen2_5VL7B => {
let model = Qwen2_5VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen2_5VL(model)
}
@@ -320,19 +340,19 @@ pub fn load_model<'a>(
let model = Qwen3AsrGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3ASR(model)
}
WhichModel::Qwen3vl2B => {
WhichModel::Qwen3VL2B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(Box::new(model))
}
WhichModel::Qwen3vl4B => {
WhichModel::Qwen3VL4B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(Box::new(model))
}
WhichModel::Qwen3vl8B => {
WhichModel::Qwen3VL8B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(Box::new(model))
}
WhichModel::Qwen3vl32B => {
WhichModel::Qwen3VL32B => {
let model = Qwen3VLGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3VL(Box::new(model))
}
src/utils/tensor_utils.rs
+22
View File
@@ -1,3 +1,5 @@
use std::f32;
use anyhow::{Result, anyhow};
use candle_core::{D, DType, Device, IndexOp, Tensor, shape::Dim};
use candle_nn::ops::sigmoid;
@@ -32,6 +34,26 @@ pub fn attn_masked_fill(on_true: &Tensor, mask: &Tensor, on_false: f32) -> Resul
Ok(filled)
}
pub fn prepare_mask(mask: &Tensor) -> Result<Tensor> {
//(bs, seq_len)
// [[1, 1, 1, 1, 0, 0]]
// ->
// [[1, 1, 1, 1, 0, 0],
// [1, 1, 1, 1, 0, 0],
// [1, 1, 1, 1, 0, 0],
// [1, 1, 1, 1, 0, 0],
// [1, 1, 1, 1, 0, 0],
// [1, 1, 1, 1, 0, 0],]
// (bs, 1, 1, seq_len)
let seq_len = mask.dim(1)?;
let mask = mask.unsqueeze(1)?.unsqueeze(1)?;
let mask = mask.repeat((1, 1, seq_len, 1))?;
let on_true = mask.zeros_like()?.to_dtype(DType::F32)?;
let on_false = Tensor::new(f32::NEG_INFINITY, mask.device())?.broadcast_as(mask.shape())?;
let mask = mask.where_cond(&on_true, &on_false)?;
Ok(mask)
}
pub fn prepare_causal_attention_mask(
b_size: usize,
tgt_len: usize,
tests/config_tests.rs
+12 -3
View File
@@ -1,5 +1,13 @@
use aha::models::{
deepseek_ocr::config::DeepseekOCRConfig, hunyuan_ocr::config::HunYuanVLConfig, lfm2::config::Lfm2Config, lfm2vl::config::{Lfm2ProcessorConfig, Lfm2VLConfig}, minicpm4::config::MiniCPM4Config, paddleocr_vl::config::PaddleOCRVLConfig, qwen2_5vl::config::Qwen2_5VLConfig, qwen3vl::config::Qwen3VLConfig, voxcpm::config::VoxCPMConfig
deepseek_ocr::config::DeepseekOCRConfig,
hunyuan_ocr::config::HunYuanVLConfig,
lfm2::config::Lfm2Config,
lfm2vl::config::{Lfm2ProcessorConfig, Lfm2VLConfig},
minicpm4::config::MiniCPM4Config,
paddleocr_vl::config::PaddleOCRVLConfig,
qwen2_5vl::config::Qwen2_5VLConfig,
qwen3vl::config::Qwen3VLConfig,
voxcpm::config::VoxCPMConfig,
};
use anyhow::Result;
@@ -104,7 +112,8 @@ fn lfm2vl_config() -> Result<()> {
let config: Lfm2VLConfig = serde_json::from_slice(&std::fs::read(config_path)?)?;
println!("{:?}", config);
let processor_config_path = model_path.to_string() + "/processor_config.json";
let processor_config: Lfm2ProcessorConfig = serde_json::from_slice(&std::fs::read(processor_config_path)?)?;
let processor_config: Lfm2ProcessorConfig =
serde_json::from_slice(&std::fs::read(processor_config_path)?)?;
println!("{:?}", processor_config);
Ok(())
}
}
tests/test_health_models.rs
+8 -8
View File
@@ -8,8 +8,8 @@ fn test_model_type_classification() {
// Since get_model_type and get_model_id are private to api.rs,
// we document the expected behavior here for reference:
//
// LLM models: MiniCPM4_0_5B, Qwen2_5vl3B, Qwen2_5vl7B, Qwen3_0_6B,
// Qwen3vl2B, Qwen3vl4B, Qwen3vl8B, Qwen3vl32B
// LLM models: MiniCPM4_0_5B, Qwen2_5VL3B, Qwen2_5VL7B, Qwen3_0_6B,
// Qwen3VL2B, Qwen3VL4B, Qwen3VL8B, Qwen3VL32B
// OCR models: DeepSeekOCR, HunyuanOCR, PaddleOCRVL
// ASR models: Qwen3ASR0_6B, Qwen3ASR1_7B, GlmASRNano2512, FunASRNano2512
// Image models: RMBG2_0, VoxCPM, VoxCPM1_5
@@ -17,13 +17,13 @@ fn test_model_type_classification() {
// This test documents the expected model type classification
let llm_models = [
WhichModel::MiniCPM4_0_5B,
WhichModel::Qwen2_5vl3B,
WhichModel::Qwen2_5vl7B,
WhichModel::Qwen2_5VL3B,
WhichModel::Qwen2_5VL7B,
WhichModel::Qwen3_0_6B,
WhichModel::Qwen3vl2B,
WhichModel::Qwen3vl4B,
WhichModel::Qwen3vl8B,
WhichModel::Qwen3vl32B,
WhichModel::Qwen3VL2B,
WhichModel::Qwen3VL4B,
WhichModel::Qwen3VL8B,
WhichModel::Qwen3VL32B,
];
let ocr_models = [
tests/test_lfm2.rs
+3 -4
View File
@@ -9,19 +9,18 @@ use std::{pin::pin, time::Instant};
#[test]
fn lfm2_generate() -> Result<()> {
// test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_lfm2 lfm2_generate -r -- --nocapture
// test with cuda+flash-attn: RUST_BACKTRACE=1 cargo test -F cuda,flash-attn qwen3_0_6b_generate -r -- --nocapture
let save_dir =
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
// let model_path = format!("{}/LiquidAI/LFM2-1.2B/", save_dir);
let model_path = format!("{}/LiquidAI/LFM2.5-1.2B-Instruct/", save_dir);
let model_path = format!("{}/LiquidAI/LFM2-1.2B/", save_dir);
// let model_path = format!("{}/LiquidAI/LFM2.5-1.2B-Instruct/", save_dir);
let message = r#"
{
"model": "lfm2",
"messages": [
{
"role": "user",
"content": "你如何看待AI"
"content": "你是谁，你如何看待AI"
}
]
}
tests/test_lfm2vl.rs
+68 -13
View File
@@ -1,13 +1,71 @@
use std::time::Instant;
use std::{pin::pin, time::Instant};
use aha::{chat::ChatCompletionParameters, models::lfm2vl::generate::Lfm2VLGenerateModel};
use aha::{
chat::ChatCompletionParameters,
models::{GenerateModel, lfm2vl::generate::Lfm2VLGenerateModel},
};
use anyhow::Result;
use rocket::futures::StreamExt;
#[test]
fn lfm2vl_generate() -> Result<()> {
// test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_lfm2vl lfm2vl_generate -r -- --nocapture
let save_dir =
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
// let model_path = format!("{}/LiquidAI/LFM2.5-VL-1.6B/", save_dir);
let model_path = format!("{}/LiquidAI/LFM2-VL-1.6B/", save_dir);
let message = r#"
{
"model": "lfm2vl",
"messages": [
{
"role": "user",
"content": [
{
"type": "image",
"image_url":
{
"url": "file://./assets/img/ocr_test1.png"
}
},
{
"type": "text",
"text": "图片里面是什么"
}
]
}
]
}
"#;
let mes: ChatCompletionParameters = serde_json::from_str(message)?;
let i_start = Instant::now();
let mut model = Lfm2VLGenerateModel::init(&model_path, None, None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
let i_start = Instant::now();
let result = model.generate(mes)?;
let i_duration = i_start.elapsed();
println!("generate: \n {:?}", result);
if let Some(usage) = &result.usage {
let num_token = usage.total_tokens;
let duration_secs = i_duration.as_secs_f64();
let tps = num_token as f64 / duration_secs;
println!("Tokens per second (TPS): {:.2}", tps);
}
println!("Time elapsed in generate is: {:?}", i_duration);
Ok(())
}
#[tokio::test]
async fn lfm2vl_stream() -> Result<()> {
// test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_lfm2vl lfm2vl_stream -r -- --nocapture
// test with cuda+flash-attn: RUST_BACKTRACE=1 cargo test -F cuda,flash-attn qwen3_0_6b_generate -r -- --nocapture
let save_dir =
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
// let model_path = format!("{}/LiquidAI/LFM2-1.2B/", save_dir);
let model_path = format!("{}/LiquidAI/LFM2.5-VL-1.6B/", save_dir);
let message = r#"
{
@@ -39,16 +97,13 @@ fn lfm2vl_generate() -> Result<()> {
println!("Time elapsed in load model is: {:?}", i_duration);
let i_start = Instant::now();
let result = model.generate(mes)?;
// let result = model.generate(mes)?;
let mut stream = pin!(model.generate_stream(mes)?);
let i_duration = i_start.elapsed();
// println!("generate: \n {:?}", result);
// if let Some(usage) = &result.usage {
// let num_token = usage.total_tokens;
// let duration_secs = i_duration.as_secs_f64();
// let tps = num_token as f64 / duration_secs;
// println!("Tokens per second (TPS): {:.2}", tps);
// }
// println!("Time elapsed in generate is: {:?}", i_duration);
while let Some(token) = stream.next().await {
println!("generate: \n {:?}", token);
}
println!("Time elapsed in generate is: {:?}", i_duration);
Ok(())
}
}
tests/weight_test.rs
+23
View File
@@ -287,3 +287,26 @@ fn lfm2_weight() -> Result<()> {
println!("model_list: {:?}", model_list);
Ok(())
}
#[test]
fn lfm2vl_weight() -> Result<()> {
// cargo test -F cuda --test weight_test lfm2vl_weight -r -- --nocapture
let save_dir =
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
// let model_path = format!("{}/LiquidAI/LFM2.5-VL-1.6B/", save_dir);
let model_path = format!("{}/LiquidAI/LFM2-VL-1.6B/", save_dir);
let model_list = find_type_files(&model_path, "safetensors")?;
let device = Device::Cpu;
for m in &model_list {
let weights = safetensors::load(m, &device)?;
for (key, tensor) in weights.iter() {
// if key.contains("lm_head") {
// println!("=== {} === {:?}", key, tensor.shape());
// }
println!("=== {} === {:?}", key, tensor.shape());
}
}
println!("model_list: {:?}", model_list);
Ok(())
}