diff --git a/Cargo.toml b/Cargo.toml index fd78630..aff9258 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ rocket = { version = "0.5.1", features = ["serde_json", "json"] } tokio = "1.47.1" hound = "3.5.1" clap = { version = "4.5.51", features = ["derive"] } -modelscope = "0.1.3" +modelscope = "0.1.4" dirs = "6.0.0" sysinfo = "0.33" url = "2.5.7" diff --git a/README.md b/README.md index b91c663..266ae12 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,13 @@ aha is a high-performance, cross-platform AI inference engine built with Rust and the Candle framework. It brings state-of-the-art AI models to your local machine—no API keys, no cloud dependencies, just pure, fast AI running directly on your hardware. ## Changelog +### 2026-03-17 +- fix qwen3.5 position_ids create bug +- cli param add + - gguf_path: Local GGUF model weight path (required for loading models with GGUF) + - mmproj_path: Local path to mmproj GGUF weights (required for multimodal GGUF loading) +- WhichModel add qwen3.5-gguf + ### 2026-03-16 - Added Qwen3.5 mmproj @@ -41,28 +48,6 @@ aha is a high-performance, cross-platform AI inference engine built with Rust an ### 2026-03-01 - update interpolate.rs -### 2026-02-24 -- update candle version 0.9.2 - -### v0.2.0 (2026-02-05) -- Added Qwen3-ASR speech recognition model - -### v0.1.9 (2026-01-31) -- Added CLI `list` subcommand to show supported models -- Added CLI subcommand structure support (`cli`, `serv`, `download`, `run`) -- Fixed Qwen3VL thinking startswith bug -- Fixed `aha run` multiple inputs bug - -### v0.1.8 (2026-01-17) -- Added Qwen3 text model support -- Added Fun-ASR-Nano-2512 speech recognition model -- Fixed ModelScope Fun-ASR-Nano model load error -- Updated audio resampling with rubato - -### v0.1.7 (2026-01-07) -- Added GLM-ASR-Nano-2512 speech recognition model -- Merged Metal (GPU) support for Apple Silicon -- Added dynamic home directory and model download script **[View full changelog](docs/changelog.md)** → diff --git a/README.zh-CN.md b/README.zh-CN.md index 771f89d..c2a9fc4 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -25,6 +25,13 @@ aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理引擎。将最先进的 AI 模型带到您的本地机器——无需 API 密钥，无需云依赖，纯粹、快速的 AI，直接在您的硬件上运行。 ## 更新日志 +### 2026-03-17 +- 修复 qwen3.5 position_ids 创建错误 +- cli 参数增加 + - gguf_path: 本地 GGUF 模型权重路径（加载 GGUF 模型时需要） + - mmproj_path: 本地 mmproj GGUF 权重路径（加载多模态 GGUF 时需要） +- WhichModel 增加 qwen3.5-gguf + ### 2026-03-16 - 增加 Qwen3.5 mmproj @@ -41,29 +48,6 @@ aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理 ### 2026-03-01 - 更新 interpolate.rs -### 2026-02-24 -- 更新 candle 版本 0.9.2 - -### v0.2.0 (2026-02-05) -- 新增 Qwen3-ASR 语音识别模型 - -### v0.1.9 (2026-01-31) -- 新增 CLI `list` 子命令，显示支持的模型 -- 新增 CLI 子命令结构支持（`cli`、`serv`、`download`、`run`） -- 修复 Qwen3VL thinking startswith bug -- 修复 `aha run` 多输入 bug - -### v0.1.8 (2026-01-17) -- 新增 Qwen3 文本模型支持 -- 新增 Fun-ASR-Nano-2512 语音识别模型 -- 修复 ModelScope Fun-ASR-Nano 模型加载错误 -- 使用 rubato 更新音频重采样 - -### v0.1.7 (2026-01-07) -- 新增 GLM-ASR-Nano-2512 语音识别模型 -- 合并 Metal (GPU) 支持，适用于 Apple Silicon -- 新增动态主目录和模型下载脚本 - **[查看完整更新日志](docs/changelog.zh-CN.md)** → ## 快速开始 diff --git a/docs/changelog.md b/docs/changelog.md index 5f9f760..a12c9e2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,13 @@ All notable changes to aha will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +### 2026-03-17 +- fix qwen3.5 position_ids create bug +- cli param add + - gguf_path: Local GGUF model weight path (required for loading models with GGUF) + - mmproj_path: Local path to mmproj GGUF weights (required for multimodal model GGUF loading) +- WhichModel add qwen3.5-gguf + ### 2026-03-16 - Added Qwen3.5 mmproj diff --git a/docs/changelog.zh-CN.md b/docs/changelog.zh-CN.md index 3fbff27..e57a2f6 100644 --- a/docs/changelog.zh-CN.md +++ b/docs/changelog.zh-CN.md @@ -5,6 +5,13 @@ 格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)， 本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/spec/v2.0.0.html)。 +### 2026-03-17 +- 修复 qwen3.5 position_ids 创建错误 +- cli 参数增加 + - gguf_path: 本地 GGUF 模型权重路径（加载 GGUF 模型时需要） + - mmproj_path: 本地 mmproj GGUF 权重路径（加载多模态 GGUF 时需要） +- WhichModel 增加 qwen3.5-gguf + ### 2026-03-16 - 增加 Qwen3.5 mmproj diff --git a/docs/cli.md b/docs/cli.md index d2cd0c0..a357d1b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -18,6 +18,8 @@ aha [COMMAND] [OPTIONS] | `--weight-path ` | Local model weight path | - | | `--save-dir ` | Model download save directory | ~/.aha/ | | `--download-retries ` | Download retry count | 3 | +| `--gguf-path ` | Local GGUF weight | - | +| `--mmproj-path ` | Local mmproj GGUF weight | - | | `-h, --help` | Display help information | - | | `-V, --version` | Display version number | - | @@ -42,6 +44,8 @@ aha cli [OPTIONS] --model | `--weight-path ` | Local model weight path (skip download if specified) | - | | `--save-dir ` | Model download save directory | ~/.aha/ | | `--download-retries ` | Download retry count | 3 | +| `--gguf-path ` | Local GGUF weight | - | +| `--mmproj-path ` | Local mmproj GGUF weight | - | **Examples:** @@ -57,6 +61,9 @@ aha cli -m qwen3vl-2b --weight-path /path/to/model # Backward compatible way (equivalent to cli subcommand) aha -m qwen3vl-2b + +# use gguf-path and mmproj-path +aha cli -m qwen3.5-gguf --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf ``` ### run - Direct model inference @@ -65,7 +72,7 @@ Run model inference directly without starting an HTTP service. Suitable for one- **Syntax:** ```bash -aha run [OPTIONS] --model --input [--input ] --weight-path +aha run [OPTIONS] --model --input [--input ] [--weight-path ] [--gguf-path ] [--mmproj-path ] ``` **Options:** @@ -75,8 +82,9 @@ aha run [OPTIONS] --model --input [--input ] --weight-pa | `-m, --model ` | Model type (required) | - | | `-i, --input ` | Input text or file path (model-specific interpretation, supports 1-2 parameters: input1: prompt text, input2: file path) | - | | `-o, --output ` | Output file path (optional, auto-generated if not specified) | - | -| `--weight-path ` | Local model weight path (required) | - | - +| `--weight-path ` | Local model weight path (required when using non-GGUF models) | - | +| `--gguf-path ` | Local GGUF model weight path（required when using GGUF models） | - | +| `--mmproj-path ` | Local mmproj GGUF weight path（optional，If not specified, the module will not be loaded） | - | **Examples:** ```bash @@ -109,6 +117,14 @@ aha run -m qwen2.5vl-3b -i "请分析图片并提取所有可见文本内容， # Qwen3-ASR speech recognition (single input: audio file) aha run -m qwen3asr-0.6b -i "audio.wav" --weight-path /path/to/model + +# Qwen3.5-GGUF without mmproj (single input: prompt text) +aha run -m qwen3.5-gguf -i 你如何看待AI --gguf-path /path/to/xxx.gguf + +# Qwen3.5-GGUF with mmproj (two inputs：prompt text + file) +aha run -m qwen3.5-gguf -i 提取图片中的文本 -i https://ai.bdstatic.com/file/C56CC9B274CF460CA33 +63E59ECD94423 --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf + ``` ### serv - Start service @@ -117,7 +133,7 @@ Start HTTP service with a model. The `--weight-path` is optional - if not specif **Syntax:** ```bash -aha serv [OPTIONS] --model [--weight-path ] +aha serv [OPTIONS] --model [--weight-path ] [--gguf-path ] [--mmproj-path ] ``` **Options:** @@ -129,6 +145,8 @@ aha serv [OPTIONS] --model [--weight-path ] | `-m, --model ` | Model type (required) | - | | `--weight-path ` | Local model weight path (optional) | ~/.aha/{model_id} | | `--allow-remote-shutdown` | Allow remote shutdown requests (not recommended) | false | +| `--gguf-path ` | Local GGUF model weight path（required when using GGUF models） | - | +| `--mmproj-path ` | Local mmproj GGUF weight path（optional，If not specified, the module will not be loaded） | - | **Examples:** diff --git a/docs/cli.zh-CN.md b/docs/cli.zh-CN.md index db83be4..5586918 100644 --- a/docs/cli.zh-CN.md +++ b/docs/cli.zh-CN.md @@ -18,6 +18,8 @@ aha [COMMAND] [OPTIONS] | `--weight-path ` | 本地模型权重路径 | - | | `--save-dir ` | 模型下载保存目录 | ~/.aha/ | | `--download-retries ` | 下载重试次数 | 3 | +| `--gguf-path ` | 本地 GGUF 模型权重 | - | +| `--mmproj-path ` | 本地 mmproj GGUF 模型权重 | - | | `-h, --help` | 显示帮助信息 | - | | `-V, --version` | 显示版本号 | - | @@ -42,6 +44,8 @@ aha cli [OPTIONS] --model | `--weight-path ` | 本地模型权重路径（如指定则跳过下载） | - | | `--save-dir ` | 模型下载保存目录 | ~/.aha/ | | `--download-retries ` | 下载重试次数 | 3 | +| `--gguf-path ` | 本地 GGUF 模型权重 | - | +| `--mmproj-path ` | 本地 mmproj GGUF 模型权重 | - | **示例：** @@ -57,6 +61,9 @@ aha cli -m qwen3vl-2b --weight-path /path/to/model # 向后兼容方式（等同于 cli 子命令） aha -m qwen3vl-2b + +# 指定gguf-path和mmproj-path +aha cli -m qwen3.5-gguf --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf ``` ### run - 直接模型推理 @@ -65,7 +72,7 @@ aha -m qwen3vl-2b **语法：** ```bash -aha run [OPTIONS] --model --input [--input ] --weight-path +aha run [OPTIONS] --model --input [--input ] [--weight-path ] [--gguf-path ] [--mmproj-path ] ``` **选项：** @@ -75,7 +82,9 @@ aha run [OPTIONS] --model --input [--input ] --weight-pa | `-m, --model ` | 模型类型（必选） | - | | `-i, --input ` | 输入文本或文件路径（模型特定解释，支持1-2个参数, input1： 提示文本, input2: 文件地址） | - | | `-o, --output ` | 输出文件路径（可选，未指定则自动生成） | - | -| `--weight-path ` | 本地模型权重路径（必选） | - | +| `--weight-path ` | 本地模型权重路径（使用非GGUF模型时必选） | - | +| `--gguf-path ` | 本地GGUF模型权重路径（使用GGUF模型时必选） | - | +| `--mmproj-path ` | 本地mmproj GGUF模型权重路径（可选，未指定则不加载该模块） | - | **示例：** @@ -109,6 +118,13 @@ aha run -m qwen2.5vl-3b -i "请分析图片并提取所有可见文本内容， # Qwen3-ASR 语音识别（单个输入：音频文件） aha run -m qwen3asr-0.6b -i "audio.wav" --weight-path /path/to/model + +# Qwen3.5-GGUF 无mmproj (单个输入：提示文本) +aha run -m qwen3.5-gguf -i 你如何看待AI --gguf-path /path/to/xxx.gguf + +# Qwen3.5-GGUF 有mmproj (两个输入：提示文本 + 文件) +aha run -m qwen3.5-gguf -i 提取图片中的文本 -i https://ai.bdstatic.com/file/C56CC9B274CF460CA33 +63E59ECD94423 --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf ``` ### serv - 启动服务 @@ -117,7 +133,7 @@ aha run -m qwen3asr-0.6b -i "audio.wav" --weight-path /path/to/model **语法：** ```bash -aha serv [OPTIONS] --model [--weight-path ] +aha serv [OPTIONS] --model [--weight-path ] [--gguf-path ] [--mmproj-path ] ``` **选项：** @@ -129,6 +145,8 @@ aha serv [OPTIONS] --model [--weight-path ] | `-m, --model ` | 模型类型（必选） | - | | `--weight-path ` | 本地模型权重路径（可选） | ~/.aha/{model_id} | | `--allow-remote-shutdown` | 允许远程关机请求（不推荐） | false | +| `--gguf-path ` | 本地GGUF模型权重路径（使用GGUF模型时必选） | - | +| `--mmproj-path ` | 本地mmproj GGUF模型权重路径（可选，未指定则不加载该模块） | - | **示例：** diff --git a/src/api/mod.rs b/src/api/mod.rs index 73f0451..f8181f0 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -37,9 +37,16 @@ static SHUTDOWN_FLAG: OnceLock> = OnceLock::new(); static SERVER_PORT: OnceLock = OnceLock::new(); static ALLOW_REMOTE_SHUTDOWN: OnceLock = OnceLock::new(); -pub fn init(model_type: WhichModel, path: String) -> anyhow::Result<()> { +pub fn init( + model_type: WhichModel, + path: String, + gguf: Option, + mmproj: Option, +) -> anyhow::Result<()> { let model_path = string_to_static_str(path); - let model = load_model(model_type, model_path)?; + let gguf = gguf.map(string_to_static_str); + let mmproj = mmproj.map(string_to_static_str); + let model = load_model(model_type, model_path, gguf, mmproj)?; MODEL.get_or_init(|| { Arc::new(RwLock::new(StoredModel { which_model: model_type, @@ -247,6 +254,7 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str { WhichModel::Qwen3_5_2B => "qwen3.5-2b", WhichModel::Qwen3_5_4B => "qwen3.5-4b", WhichModel::Qwen3_5_9B => "qwen3.5-9b", + WhichModel::Qwen3_5Gguf => "qwen3.5-gguf", WhichModel::Qwen3ASR0_6B => "qwen3asr-0.6b", WhichModel::Qwen3ASR1_7B => "qwen3asr-1.7b", WhichModel::Qwen3vl2B => "qwen3vl-2b", @@ -274,7 +282,8 @@ fn which_model_to_owner(which_model: WhichModel) -> &'static str { WhichModel::Qwen3vl2B | WhichModel::Qwen3vl4B | WhichModel::Qwen3vl8B - | WhichModel::Qwen3vl32B => "Qwen", + | WhichModel::Qwen3vl32B + | WhichModel::Qwen3_5Gguf => "Qwen", WhichModel::Qwen3_5_0_8B | WhichModel::Qwen3_5_2B | WhichModel::Qwen3_5_4B diff --git a/src/exec/qwen3_5.rs b/src/exec/qwen3_5.rs index fd0cd86..c69e5a8 100644 --- a/src/exec/qwen3_5.rs +++ b/src/exec/qwen3_5.rs @@ -2,15 +2,138 @@ use std::time::Instant; -use anyhow::Result; +use anyhow::{Result, anyhow}; use crate::exec::ExecModel; use crate::models::GenerateModel; use crate::models::qwen3_5::generate::Qwen3_5GenerateModel; -use crate::utils::get_file_path; +use crate::utils::{get_file_path, string_to_static_str}; pub struct Qwen3_5Exec; +impl Qwen3_5Exec { + pub fn run_gguf( + input: &[String], + output: Option<&str>, + gguf_path: Option, + mmproj_path: Option, + ) -> Result<()> { + let input_text = &input[0]; + let target_text = if input_text.starts_with("file://") { + let path = get_file_path(input_text)?; + std::fs::read_to_string(path)? + } else { + input_text.clone() + }; + let model_file = if let Some(g) = gguf_path { + g + } else { + return Err(anyhow!("gguf model path is required")); + }; + let mmproj_path = mmproj_path.map(string_to_static_str); + + let i_start = Instant::now(); + let mut model = Qwen3_5GenerateModel::init_from_gguf(&model_file, mmproj_path, None)?; + let i_duration = i_start.elapsed(); + println!("Time elapsed in load model is: {:?}", i_duration); + let url = input.get(1); + let input_url = if let Some(url) = url + && (url.starts_with("http://") + || url.starts_with("https://") + || url.starts_with("file://")) + { + Some(url.clone()) + } else { + url.map(|url| format!("file://{}", url)) + }; + let message = if let Some(input_url) = &input_url + && input_url.ends_with("mp4") + { + format!( + r#"{{ + "model": "qwen3.5", + "messages": [ + {{ + "role": "user", + "content": [ + {{ + "type": "video", + "video_url": + {{ + "url": "{}" + }} + }}, + {{ + "type": "text", + "text": "{}" + }} + ] + }} + ] + }}"#, + input_url, target_text + ) + } else if let Some(input_url) = &input_url { + format!( + r#"{{ + "model": "qwen3.5", + "messages": [ + {{ + "role": "user", + "content": [ + {{ + "type": "image", + "image_url": {{ + "url": "{}" + }} + }}, + {{ + "type": "text", + "text": "{}" + }} + ] + }} + ] + }}"#, + input_url, target_text + ) + } else { + format!( + r#"{{ + "model": "qwen3.5", + "messages": [ + {{ + "role": "user", + "content": [ + {{ + "type": "text", + "text": "{}" + }} + ] + }} + ] + }}"#, + target_text + ) + }; + let mes = serde_json::from_str(&message)?; + + let i_start = Instant::now(); + let result = model.generate(mes)?; + let i_duration = i_start.elapsed(); + println!("Time elapsed in generate is: {:?}", i_duration); + + println!("Result: {:?}", result); + + if let Some(out) = output { + std::fs::write(out, format!("{:?}", result))?; + println!("Output saved to: {}", out); + } + + Ok(()) + } +} + impl ExecModel for Qwen3_5Exec { fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> { let input_text = &input[0]; @@ -62,7 +185,7 @@ impl ExecModel for Qwen3_5Exec { } else { format!( r#"{{ - "model": "qwen2.5", + "model": "qwen3.5", "messages": [ {{ "role": "user", diff --git a/src/exec/qwen3vl.rs b/src/exec/qwen3vl.rs index 493a170..26c1c0f 100644 --- a/src/exec/qwen3vl.rs +++ b/src/exec/qwen3vl.rs @@ -24,16 +24,19 @@ impl ExecModel for Qwen3vlExec { let mut model = Qwen3VLGenerateModel::init(weight_path, None, None)?; let i_duration = i_start.elapsed(); println!("Time elapsed in load model is: {:?}", i_duration); - let url = &input[1]; - let input_url = if url.starts_with("http://") - || url.starts_with("https://") - || url.starts_with("file://") + let url = input.get(1); + let input_url = if let Some(url) = url + && (url.starts_with("http://") + || url.starts_with("https://") + || url.starts_with("file://")) { - url.clone() + Some(url.clone()) } else { - format!("file://{}", url) + url.map(|url| format!("file://{}", url)) }; - let message = if input_url.ends_with("mp4") { + let message = if let Some(input_url) = &input_url + && input_url.ends_with("mp4") + { format!( r#"{{ "model": "qwen3vl", @@ -58,7 +61,7 @@ impl ExecModel for Qwen3vlExec { }}"#, input_url, target_text ) - } else { + } else if let Some(input_url) = &input_url { format!( r#"{{ "model": "qwen3vl", @@ -82,6 +85,24 @@ impl ExecModel for Qwen3vlExec { }}"#, input_url, target_text ) + } else { + format!( + r#"{{ + "model": "qwen3vl", + "messages": [ + {{ + "role": "user", + "content": [ + {{ + "type": "text", + "text": "{}" + }} + ] + }} + ] + }}"#, + target_text + ) }; let mes = serde_json::from_str(&message)?; diff --git a/src/main.rs b/src/main.rs index 1455e41..27ec4a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ use aha::{ process::{cleanup_pid_file, create_pid_file}, utils::{download_model, get_default_save_dir}, }; +use anyhow::anyhow; use clap::{Args, Parser, Subcommand, ValueEnum}; use rocket::{ Config, @@ -45,6 +46,14 @@ struct Cli { #[arg(long)] download_retries: Option, + /// Local GGUF model weight path (required for loading models with GGUF). + #[arg(long)] + gguf_path: Option, + + /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) + #[arg(long)] + mmproj_path: Option, + #[command(subcommand)] command: Option, } @@ -104,6 +113,14 @@ struct CliArgs { /// Download retry count #[arg(long)] download_retries: Option, + + /// Local GGUF model weight path (required for loading models with GGUF). + #[arg(long)] + gguf_path: Option, + + /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) + #[arg(long)] + mmproj_path: Option, } /// Arguments for the 'serv start' subcommand @@ -115,6 +132,14 @@ struct ServArgs { /// Local model weight path (defaults to ~/.aha/{model_id} if not specified) #[arg(long)] weight_path: Option, + + /// Local GGUF model weight path (required for loading models with GGUF). + #[arg(long)] + gguf_path: Option, + + /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) + #[arg(long)] + mmproj_path: Option, } /// Arguments for the 'serv list' subcommand @@ -159,6 +184,14 @@ struct RunArgs { /// Local model weight path (defaults to ~/.aha/{model_id} if not specified) #[arg(long)] weight_path: Option, + + /// Local GGUF model weight path (required for loading models with GGUF). + #[arg(long)] + gguf_path: Option, + + /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) + #[arg(long)] + mmproj_path: Option, } /// Arguments for the 'delete' subcommand (delete model from default location) @@ -282,23 +315,33 @@ async fn run_cli(args: CliArgs) -> anyhow::Result<()> { weight_path, save_dir, download_retries, + gguf_path, + mmproj_path, } = args; let model_id = common.model.model_id(); - let model_path = match weight_path { - Some(path) => path, - None => { - let save_dir = match save_dir { - Some(dir) => dir, - None => get_default_save_dir().expect("Failed to get home directory"), - }; - let max_retries = download_retries.unwrap_or(3); - download_model(model_id, &save_dir, max_retries).await?; - save_dir + "/" + model_id + let (model_path, gguf, mmproj) = if model_id.eq("GGUF") { + if gguf_path.is_none() { + return Err(anyhow!("gguf model path is required")); } + ("GGUF".to_string(), gguf_path, mmproj_path) + } else { + let model_path = match weight_path { + Some(path) => path, + None => { + let save_dir = match save_dir { + Some(dir) => dir, + None => get_default_save_dir().expect("Failed to get home directory"), + }; + let max_retries = download_retries.unwrap_or(3); + download_model(model_id, &save_dir, max_retries).await?; + save_dir + "/" + model_id + } + }; + (model_path, None, None) }; - init(common.model, model_path)?; + init(common.model, model_path, gguf, mmproj)?; start_http_server(common.address, common.port, common.allow_remote_shutdown).await?; Ok(()) @@ -309,14 +352,24 @@ async fn run_serv(args: ServArgs) -> anyhow::Result<()> { let ServArgs { common, weight_path, + gguf_path, + mmproj_path, } = args; - - let model_path = match weight_path { - Some(path) => path, - None => get_default_weight_path(common.model), + let model_id = common.model.model_id(); + let (model_path, gguf, mmproj) = if model_id.eq("GGUF") { + if gguf_path.is_none() { + return Err(anyhow!("gguf model path is required")); + } + ("GGUF".to_string(), gguf_path, mmproj_path) + } else { + let model_path = match weight_path { + Some(path) => path, + None => get_default_weight_path(common.model), + }; + (model_path, None, None) }; - init(common.model, model_path)?; + init(common.model, model_path, gguf, mmproj)?; start_http_server(common.address, common.port, common.allow_remote_shutdown).await?; Ok(()) @@ -392,6 +445,8 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> { input, output, weight_path, + gguf_path, + mmproj_path, } = args; // Use default weight path if not specified @@ -399,7 +454,6 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> { Some(path) => path, None => get_default_weight_path(model), }; - match model { WhichModel::MiniCPM4_0_5B => { use aha::exec::minicpm4::MiniCPM4Exec; @@ -433,6 +487,10 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> { use aha::exec::qwen3_5::Qwen3_5Exec; Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?; } + WhichModel::Qwen3_5Gguf => { + use aha::exec::qwen3_5::Qwen3_5Exec; + Qwen3_5Exec::run_gguf(&input, output.as_deref(), gguf_path, mmproj_path)?; + } WhichModel::Qwen3ASR0_6B => { use aha::exec::qwen3_asr::Qwen3ASRExec; Qwen3ASRExec::run(&input, output.as_deref(), &weight_path)?; @@ -610,6 +668,8 @@ async fn main() -> anyhow::Result<()> { weight_path: cli.weight_path, save_dir: cli.save_dir, download_retries: cli.download_retries, + gguf_path: cli.gguf_path, + mmproj_path: cli.mmproj_path, }; run_cli(args).await } diff --git a/src/models/common/gguf.rs b/src/models/common/gguf.rs index 684734d..86103a8 100644 --- a/src/models/common/gguf.rs +++ b/src/models/common/gguf.rs @@ -213,6 +213,10 @@ impl QuantizedLinear { pub fn new(inner: QMatMul, bias: Option) -> Self { Self { inner, bias } } + + pub fn inner_dequantize(&self) -> Result { + Ok(self.inner.dequantize_f16()?) + } } impl Module for QuantizedLinear { diff --git a/src/models/mod.rs b/src/models/mod.rs index 10c1f98..2e6ea3a 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -22,7 +22,7 @@ pub mod w2v_bert_2_0; use aha_openai_dive::v1::resources::chat::{ ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse, }; -use anyhow::Result; +use anyhow::{Result, anyhow}; use rocket::futures::Stream; use crate::models::{ @@ -54,6 +54,8 @@ pub enum WhichModel { Qwen3_5_4B, #[value(name = "qwen3.5-9b", hide = true)] Qwen3_5_9B, + #[value(name = "qwen3.5-gguf", hide = true)] + Qwen3_5Gguf, #[value(name = "qwen3asr-0.6b", hide = true)] Qwen3ASR0_6B, #[value(name = "qwen3asr-1.7b", hide = true)] @@ -98,6 +100,7 @@ impl WhichModel { WhichModel::Qwen3_5_2B => "Qwen/Qwen3.5-2B", WhichModel::Qwen3_5_4B => "Qwen/Qwen3.5-4B", WhichModel::Qwen3_5_9B => "Qwen/Qwen3.5-9B", + WhichModel::Qwen3_5Gguf => "GGUF", WhichModel::Qwen3ASR0_6B => "Qwen/Qwen3-ASR-0.6B", WhichModel::Qwen3ASR1_7B => "Qwen/Qwen3-ASR-1.7B", WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct", @@ -130,7 +133,8 @@ impl WhichModel { | WhichModel::Qwen3_5_0_8B | WhichModel::Qwen3_5_2B | WhichModel::Qwen3_5_4B - | WhichModel::Qwen3_5_9B => "vlm", + | WhichModel::Qwen3_5_9B + | WhichModel::Qwen3_5Gguf => "vlm", // OCR models WhichModel::DeepSeekOCR | WhichModel::HunyuanOCR @@ -229,7 +233,12 @@ impl<'a> GenerateModel for ModelInstance<'a> { } } -pub fn load_model(model_type: WhichModel, path: &str) -> Result> { +pub fn load_model<'a>( + model_type: WhichModel, + path: &str, + gguf: Option<&str>, + mmproj: Option<&str>, +) -> Result> { let model = match model_type { WhichModel::MiniCPM4_0_5B => { let model = MiniCPMGenerateModel::init(path, None, None)?; @@ -263,6 +272,14 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result { + if gguf.is_none() { + return Err(anyhow!("Qwen3_5Gguf gguf model path is required")); + } + let gguf = gguf.unwrap(); + let model = Qwen3_5GenerateModel::init_from_gguf(gguf, mmproj, None)?; + ModelInstance::Qwen3_5(model) + } WhichModel::Qwen3ASR0_6B => { let model = Qwen3AsrGenerateModel::init(path, None, None)?; ModelInstance::Qwen3ASR(model) diff --git a/src/models/qwen3_5/generate.rs b/src/models/qwen3_5/generate.rs index ca52310..992b450 100644 --- a/src/models/qwen3_5/generate.rs +++ b/src/models/qwen3_5/generate.rs @@ -102,7 +102,9 @@ impl<'a> Qwen3_5GenerateModel<'a> { (None, None) }; - // let eos_token_id = gguf.get_matedata("tokenizer.ggml.eos_token_id")?.to_u32()?; + let eos_token_id = model_gguf + .get_matedata("tokenizer.ggml.eos_token_id")? + .to_u32()?; let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut model_gguf, mmproj_gguf.as_mut(), &device)?; let stem = std::path::Path::new(model_file) .file_stem() // 获取文件名主干（不含扩展名） @@ -114,7 +116,8 @@ impl<'a> Qwen3_5GenerateModel<'a> { pre_processor, qwen3_5, device, - eos_token_id: 248044, + // eos_token_id: 248044, + eos_token_id, model_name: stem.to_string(), repeat_penalty: 1.1, repeat_last_n: 64, @@ -125,7 +128,7 @@ impl<'a> Qwen3_5GenerateModel<'a> { impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> { fn generate(&mut self, mes: ChatCompletionParameters) -> Result { let seed = mes.seed.unwrap_or(32768) as u64; - let temperature = mes.temperature.unwrap_or(0.6); + let temperature = mes.temperature.unwrap_or(0.4); let top_p = mes.top_p.unwrap_or(0.95); let mut logit_processor = get_logit_processor(temperature.into(), top_p.into(), Some(20), seed); @@ -144,7 +147,6 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> { } else { (mes_render, None, None, None, None) }; - // let input = self.pre_processor.process_info(&mes, &mes_render)?; let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?; let mut seq_len = input_ids.dim(1)?; let prompt_tokens = seq_len as u32; diff --git a/src/models/qwen3_5/model.rs b/src/models/qwen3_5/model.rs index 7a254d9..702f80a 100644 --- a/src/models/qwen3_5/model.rs +++ b/src/models/qwen3_5/model.rs @@ -750,7 +750,7 @@ impl Qwen3_5Attention { } pub fn clear_kv_cache(&mut self) { - self.kv_cache = None + self.kv_cache = None; } } @@ -1024,7 +1024,6 @@ impl Qwen3_5TextModel { // i += 1; } xs = self.norm.forward(&xs)?; - // println!("norm : {}", xs); Ok(xs) } @@ -1348,7 +1347,9 @@ impl Qwen3_5Model { video_grid_thw: Option<&Tensor>, seqlen_offset: usize, ) -> Result { - let position_ids = if let Some(rope_deltas) = &self.rope_deltas { + let position_ids = if let Some(rope_deltas) = &self.rope_deltas + && seqlen_offset != 0 + { let (bs, seq_len, _) = inputs_embeds.dims3()?; Tensor::arange( seqlen_offset as i64, @@ -1383,12 +1384,12 @@ impl Qwen3_5Model { seqlen_offset: usize, ) -> Result { let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?; - // println!("embed_tokens: {}", inputs_embeds); if let Some(pixel_values) = pixel_values && let Some(image_grid_thw) = image_grid_thw && let Some(visual) = self.visual.as_ref() { let (image_embeds, _) = visual.forward(pixel_values, image_grid_thw)?; + // println!("image_embeds: {}", image_embeds); let vision_mask = get_equal_mask(input_ids, self.image_token_id)?; let n_image_tokens = vision_mask.sum_all()?.to_scalar::()?; if n_image_tokens as usize != image_embeds.dim(0)? { @@ -1429,9 +1430,7 @@ impl Qwen3_5Model { let outputs = self.language_model.forward(&inputs_embeds, &position_ids)?; let seq_len = outputs.dim(1)?; let hidden_state = outputs.narrow(1, seq_len - 1, 1)?; - // println!("narrow 1 : {}", hidden_state); let logits = self.lm_head.forward(&hidden_state)?; - // println!("logits : {}", logits); Ok(logits) } diff --git a/src/models/qwen3vl/model.rs b/src/models/qwen3vl/model.rs index 1113dad..c618714 100644 --- a/src/models/qwen3vl/model.rs +++ b/src/models/qwen3vl/model.rs @@ -95,8 +95,9 @@ impl Qwen3VLVisionPatchEmbed { pub fn forward(&self, hidden_states: &Tensor) -> Result { // hidden_states shape: (grid_t*grid_h*grid_w, c*temporal_patch_size*patch_size*patch_size) // ((), 1536) matmul (1536, 1024) -> ((), 1024) - let hidden_states = hidden_states.matmul(&self.conv3d_weight)?; - let hidden_states = hidden_states.broadcast_add(&self.conv3d_bias)?; + let dtype = hidden_states.dtype(); + let hidden_states = hidden_states.matmul(&self.conv3d_weight.to_dtype(dtype)?)?; + let hidden_states = hidden_states.broadcast_add(&self.conv3d_bias.to_dtype(dtype)?)?; Ok(hidden_states) } } @@ -169,7 +170,12 @@ impl Qwen3VLVisionPatchMerger { } else { xs.clone() }; - let xs = self.norm.forward(&xs)?.reshape(((), self.hidden_size))?; + let orig_dtype = xs.dtype(); + let xs = self + .norm + .forward(&xs.to_dtype(self.norm.weight().dtype())?)? + .reshape(((), self.hidden_size))?; + let xs = xs.to_dtype(orig_dtype)?; let xs = self .linear_fc2 .forward(&self.act_fn.forward(&self.linear_fc1.forward(&xs)?)?)?; @@ -343,12 +349,21 @@ impl Qwen3VLVisionBlock { cos: &Tensor, sin: &Tensor, ) -> Result { + let orig_dtype = xs.dtype(); let residual = xs.clone(); - let xs = self.norm1.forward(xs)?; + let xs = self + .norm1 + .forward(&xs.to_dtype(self.norm1.weight().dtype())?)?; + let xs = xs.to_dtype(orig_dtype)?; let xs = self.attn.forward(&xs, cos, sin, cu_seqlens)?; let xs = (residual + xs)?; let residual = xs.clone(); - let xs = self.mlp.forward(&self.norm2.forward(&xs)?)?; + let xs = self.mlp.forward( + &self + .norm2 + .forward(&xs.to_dtype(self.norm2.weight().dtype())?)? + .to_dtype(orig_dtype)?, + )?; let xs = (residual + xs)?; Ok(xs) } @@ -679,7 +694,9 @@ impl Qwen3VLVisionModel { grid_thw: &Tensor, ) -> Result<(Tensor, Vec)> { let hidden_states = self.patch_embed.forward(hidden_states)?; - let pos_embeds = self.fast_pos_embed_interpolate(grid_thw)?; + let pos_embeds = self + .fast_pos_embed_interpolate(grid_thw)? + .to_dtype(hidden_states.dtype())?; let hidden_states = hidden_states.broadcast_add(&pos_embeds)?; let rotary_pos_emb = self.rot_pos_emb(grid_thw)?; let seq_len = hidden_states.dim(0)?; diff --git a/src/position_embed/rope.rs b/src/position_embed/rope.rs index 8e55d1f..e4bb887 100644 --- a/src/position_embed/rope.rs +++ b/src/position_embed/rope.rs @@ -520,6 +520,7 @@ impl Qwen3VLTextRotaryEmbedding { let position_ids_expanded = position_ids .unsqueeze(D::Minus2)? .to_dtype(DType::F32)? + // .to_dtype(dtype)? .contiguous()?; // inv_freq Vec -> Tensor(1, 1, head_dim / 2, 1) -> (3, bs, head_dim / 2, 1) let inv_freq_expanded = Tensor::from_vec( @@ -529,6 +530,7 @@ impl Qwen3VLTextRotaryEmbedding { )? .broadcast_as((3, position_ids.dim(1)?, self.inv_freq.len(), 1))? .to_dtype(DType::F32)? + // .to_dtype(dtype)? .contiguous()?; // (3, bs, head_dim / 2, 1) matmul (3, bs, 1, position) diff --git a/tests/messy_test.rs b/tests/messy_test.rs index 8b8d01a..83608e7 100644 --- a/tests/messy_test.rs +++ b/tests/messy_test.rs @@ -10,12 +10,32 @@ use anyhow::Result; // use byteorder::{LittleEndian, ReadBytesExt}; use candle_core::Tensor; +use modelscope::{DownloadOptions, ModelScope}; // use sentencepiece::SentencePieceProcessor; // use zip::ZipArchive; +#[tokio::test] +async fn download_test() -> Result<()> { + // cargo test -F cuda --test messy_test download_test -r -- --nocapture + let model_id = "unsloth/Qwen3.5-4B-GGUF"; + let model_name = "Qwen3.5-4B-IQ4_NL.gguf"; + let save_dir = + aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; + let _ = ModelScope::download_with_options( + model_id, + save_dir, + DownloadOptions { + files: (vec![model_name.to_string()]).into(), + }, + ) + .await; + Ok(()) +} + #[test] fn messy_test() -> Result<()> { // RUST_BACKTRACE=1 cargo test -F cuda --test messy_test messy_test -r -- --nocapture + let device = &candle_core::Device::Cpu; let t1 = Tensor::randn(0.0, 1.0, (16, 9, 64, 128), device)?; let t2 = Tensor::randn(0.0, 1.0, (16, 9, 128, 64), device)?; diff --git a/tests/test_gguf_qwen3_5.rs b/tests/test_gguf_qwen3_5.rs index c7b3c2c..05326de 100644 --- a/tests/test_gguf_qwen3_5.rs +++ b/tests/test_gguf_qwen3_5.rs @@ -9,13 +9,19 @@ use anyhow::Result; #[test] fn gguf_test() -> Result<()> { // RUST_BACKTRACE=1 cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture - // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题 + // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q6_K.gguf"; // 有问题 // let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/mmproj-F16.gguf"; // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf"; + // let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/mmproj-F16.gguf"; let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf"; let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/mmproj-F16.gguf"; // let mut model_file = std::fs::File::open(model_path)?; // let model = gguf_file::Content::read(&mut model_file)?; + // for (key, value) in model.tensor_infos { + // if key.contains("blk.12.") { + // println!("{key}: {:#?}", value); + // } + // } // for (key, value) in model.metadata { // if key.contains("tokeni") { // continue;