diff --git a/Cargo.lock b/Cargo.lock index 5f23147..cd6ca1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,7 +19,7 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aha" -version = "0.0.0" +version = "0.1.0" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 990f0c1..f42c43f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "aha" -version = "0.0.0" +version = "0.1.0" edition = "2024" repository = "https://github.com/jhqxxx/aha" license = "Apache-2.0" -description = "aha" +description = "aha model inference library, now supports Qwen2.5VL, MiniCPM4, and VoxCPM" [dependencies] candle-core = { git = "https://github.com/huggingface/candle.git", version = "0.9.1"} diff --git a/README.md b/README.md new file mode 100644 index 0000000..e1487ba --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +# aha +一个基于 Candle 框架的 Rust 模型推理库，提供高效、易用的多模态模型推理能力。 + +## 特性 +* 🚀 高性能推理 - 基于 Candle 框架，提供高效的张量计算和模型推理 +* 🎯 多模型支持 - 集成视觉、语言和多模态模型 +* 🔧 易于使用 - 简洁的 API 设计，快速上手 +* 🛡️ 内存安全 - 得益于 Rust 的所有权系统，确保内存安全 +* 📦 轻量级 - 最小化依赖，编译产物小巧 +* ⚡ GPU 加速 - 可选 CUDA 支持 +* 🧠 注意力优化 - 可选 Flash Attention 支持，优化长序列处理 + +## 支持的模型 +### 当前已实现 +* Qwen2.5VL - 阿里通义千问 2.5 多模态大语言模型 +* MiniCPM4 - 面壁智能 MiniCPM 系列语言模型 +* VoxCPM - 面壁智能语音生成模型 + +## 计划支持 +我们持续扩展支持的模型列表，欢迎贡献！ + +## 安装 +### 作为库使用 +* cargo add aha +* 或者在Cargo.toml中添加 +```toml +[dependencies] +aha = { git = "https://github.com/jhqxxx/aha.git" } + +# 启用 CUDA 支持（可选） +candle-inference = { git = "https://github.com/jhqxxx/aha.git", features = ["cuda"] } + +# 启用Flash Attention 支持（可选） +candle-inference = { git = "https://github.com/your-username/your-repo.git", features = ["cuda", "flash-attn"] } +``` + +### 从源码构建运行测试 +```bash +git clone https://github.com/jhqxxx/aha.git +cd aha +# 修改测试用例中模型路径 +# 运行 Qwen2.5VL 示例 +cargo test -F cuda,flash-attn qwen2_5vl_generate -- --nocapture + +# 运行 MiniCPM4 示例 +cargo test -F cuda,flash-attn minicpm_generate -- --nocapture + +# 运行 VoxCPM 示例 +cargo test -F cuda,flash-attn voxcpm_generate -- --nocapture +``` + +## 开发 +### 项目结构 +```text +. +├── Cargo.toml +├── README.md +├── src +│ ├── chat_template +│ ├── models +│ │ ├── common +│ │ ├── minicpm4 +│ │ ├── qwen2_5vl +│ │ ├── voxcpm +│ │ └── mod.rs +│ ├── position_embed +│ ├── tokenizer +│ ├── utils +│ └── lib.rs +└── tests + ├── test_minicpm4.rs + ├── test_qwen2_5vl.rs + └── test_voxcpm.rs +``` + +### 添加新模型 +* 在src/models/创建新模型文件 +* 在src/models/mod.rs中导出 +* 在tests/中添加测试和示例 + +## 许可证 +本项目采用 Apache License, Version 2.0 许可证 - 查看 [LICENSE](./LICENSE) 文件了解详情。 + +## 致谢 +* [Candle](https://github.com/huggingface/candle) - 优秀的 Rust 机器学习框架 +* 所有模型的原作者和贡献者 + +## 支持 +#### 如果你遇到问题： +1. 查看 Issues 是否已有解决方案 +2. 提交新的 Issue，包含详细描述和复现步骤 + +## 更新日志 +### v0.1.0 +* 初始版本发布 +* 支持 Qwen2.5VL, MiniCPM4, VoxCPM 模型 + + +⭐ 如果这个项目对你有帮助，请给我们一个 Star！ \ No newline at end of file diff --git a/assets/img/ocr_test.png b/assets/img/ocr_test.png deleted file mode 100644 index b69f68e..0000000 Binary files a/assets/img/ocr_test.png and /dev/null differ diff --git a/src/models/base_modules/mod.rs b/src/models/common/mod.rs similarity index 100% rename from src/models/base_modules/mod.rs rename to src/models/common/mod.rs diff --git a/src/models/minicpm4/model.rs b/src/models/minicpm4/model.rs index 8fe530a..eebfa14 100644 --- a/src/models/minicpm4/model.rs +++ b/src/models/minicpm4/model.rs @@ -1,6 +1,6 @@ use crate::{ models::{ - base_modules::{AttentionNobias, MLPNoBias}, + common::{AttentionNobias, MLPNoBias}, minicpm4::config::MiniCPM4Config, }, position_embed::rope::compute_default_rope_parameters, @@ -155,7 +155,7 @@ impl MiniCPMDecoderLayer { + xs.affine( self.scale_depth as f64 / (self.num_hidden_layers as f64).sqrt(), 0.0, - ))?; + )?)?; Ok(xs) } @@ -180,7 +180,7 @@ impl MiniCPMDecoderLayer { + xs.affine( self.scale_depth as f64 / (self.num_hidden_layers as f64).sqrt(), 0.0, - ))?; + )?)?; Ok(xs) } pub fn clear_kv_cache(&mut self) { diff --git a/src/models/mod.rs b/src/models/mod.rs index 1f16059..e6e58c2 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -1,4 +1,4 @@ -pub mod base_modules; +pub mod common; pub mod minicpm4; pub mod qwen2_5vl; pub mod voxcpm; diff --git a/src/models/voxcpm/minicpm4.rs b/src/models/voxcpm/minicpm4.rs index e0e0e36..666b10a 100644 --- a/src/models/voxcpm/minicpm4.rs +++ b/src/models/voxcpm/minicpm4.rs @@ -1,7 +1,7 @@ use crate::{ models::{ - base_modules::{AttentionNobias, MLPNoBias}, + common::{AttentionNobias, MLPNoBias}, voxcpm::config::VoxMiniCPM4Config, }, position_embed::rope::compute_default_rope_parameters, @@ -193,7 +193,7 @@ impl MiniCPMDecoderLayer { + xs.affine( self.scale_depth as f64 / (self.num_hidden_layers as f64).sqrt(), 0.0, - ))?; + )?)?; res_add } else { let res_add = (residual + xs)?; @@ -206,7 +206,7 @@ impl MiniCPMDecoderLayer { + xs.affine( self.scale_depth as f64 / (self.num_hidden_layers as f64).sqrt(), 0.0, - ))?; + )?)?; res_add } else { let res_add = (residual + xs)?; diff --git a/tests/test_voxcpm.rs b/tests/test_voxcpm.rs index 71d1c57..72bbea5 100644 --- a/tests/test_voxcpm.rs +++ b/tests/test_voxcpm.rs @@ -1,18 +1,14 @@ use anyhow::{Ok, Result}; -use std::{collections::HashMap, time::Instant}; +use std::{time::Instant}; use aha::{ - models::voxcpm::{ - audio_vae::AudioVAE, config::VoxCPMConfig, generate::VoxCPMGenerate, model::VoxCPMModel, + models::voxcpm::{ generate::VoxCPMGenerate, tokenizer::SingleChineseTokenizer, }, utils::{ audio_utils::save_wav, - utils::{find_type_files, get_device}, }, }; -use candle_core::pickle::read_all_with_key; -use candle_nn::VarBuilder; #[test] fn voxcpm_generate() -> Result<()> {