cli add about gguf param

Cargo.toml

@@ -28,7 +28,7 @@ rocket = { version = "0.5.1", features = ["serde_json", "json"] }
 tokio = "1.47.1"
 hound = "3.5.1"
 clap = { version = "4.5.51", features = ["derive"] }
-modelscope = "0.1.3"
+modelscope = "0.1.4"
 dirs = "6.0.0"
 sysinfo = "0.33"
 url = "2.5.7"

README.md

@@ -25,6 +25,13 @@
 aha is a high-performance, cross-platform AI inference engine built with Rust and the Candle framework. It brings state-of-the-art AI models to your local machine—no API keys, no cloud dependencies, just pure, fast AI running directly on your hardware.
 
 ## Changelog
+### 2026-03-17
+- fix qwen3.5 position_ids create bug
+- cli param add 
+  - gguf_path: Local GGUF model weight path (required for loading models with GGUF)
+  - mmproj_path: Local path to mmproj GGUF weights (required for multimodal GGUF loading)
+- WhichModel add qwen3.5-gguf
+
 ### 2026-03-16
 - Added Qwen3.5 mmproj
 
@@ -41,28 +48,6 @@ aha is a high-performance, cross-platform AI inference engine built with Rust an
 ### 2026-03-01
 - update interpolate.rs
 
-### 2026-02-24
-- update candle version 0.9.2
-
-### v0.2.0 (2026-02-05)
-- Added Qwen3-ASR speech recognition model
-
-### v0.1.9 (2026-01-31)
-- Added CLI `list` subcommand to show supported models
-- Added CLI subcommand structure support (`cli`, `serv`, `download`, `run`)
-- Fixed Qwen3VL thinking startswith bug
-- Fixed `aha run` multiple inputs bug
-
-### v0.1.8 (2026-01-17)
-- Added Qwen3 text model support
-- Added Fun-ASR-Nano-2512 speech recognition model
-- Fixed ModelScope Fun-ASR-Nano model load error
-- Updated audio resampling with rubato
-
-### v0.1.7 (2026-01-07)
-- Added GLM-ASR-Nano-2512 speech recognition model
-- Merged Metal (GPU) support for Apple Silicon
-- Added dynamic home directory and model download script
 
 **[View full changelog](docs/changelog.md)** →
 

README.zh-CN.md

@@ -25,6 +25,13 @@
 aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理引擎。将最先进的 AI 模型带到您的本地机器——无需 API 密钥，无需云依赖，纯粹、快速的 AI，直接在您的硬件上运行。
 
 ## 更新日志
+### 2026-03-17
+- 修复 qwen3.5 position_ids 创建错误
+- cli 参数增加 
+  - gguf_path: 本地 GGUF 模型权重路径（加载 GGUF 模型时需要）
+  - mmproj_path: 本地 mmproj GGUF 权重路径（加载多模态 GGUF 时需要）
+- WhichModel 增加 qwen3.5-gguf
+
 ### 2026-03-16
 - 增加 Qwen3.5 mmproj
 
@@ -41,29 +48,6 @@ aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理
 ### 2026-03-01
 - 更新 interpolate.rs
 
-### 2026-02-24
-- 更新 candle 版本 0.9.2
-
-### v0.2.0 (2026-02-05)
-- 新增 Qwen3-ASR 语音识别模型
-
-### v0.1.9 (2026-01-31)
-- 新增 CLI `list` 子命令，显示支持的模型
-- 新增 CLI 子命令结构支持（`cli`、`serv`、`download`、`run`）
-- 修复 Qwen3VL thinking startswith bug
-- 修复 `aha run` 多输入 bug
-
-### v0.1.8 (2026-01-17)
-- 新增 Qwen3 文本模型支持
-- 新增 Fun-ASR-Nano-2512 语音识别模型
-- 修复 ModelScope Fun-ASR-Nano 模型加载错误
-- 使用 rubato 更新音频重采样
-
-### v0.1.7 (2026-01-07)
-- 新增 GLM-ASR-Nano-2512 语音识别模型
-- 合并 Metal (GPU) 支持，适用于 Apple Silicon
-- 新增动态主目录和模型下载脚本
-
 **[查看完整更新日志](docs/changelog.zh-CN.md)** →
 
 ## 快速开始

docs/changelog.md

@@ -5,6 +5,13 @@ All notable changes to aha will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+### 2026-03-17
+- fix qwen3.5 position_ids create bug
+- cli param add 
+  - gguf_path: Local GGUF model weight path (required for loading models with GGUF)
+  - mmproj_path: Local path to mmproj GGUF weights (required for multimodal model GGUF loading)
+- WhichModel add qwen3.5-gguf
+
 ### 2026-03-16
 - Added Qwen3.5 mmproj
 

docs/changelog.zh-CN.md

@@ -5,6 +5,13 @@
 格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)，
 本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/spec/v2.0.0.html)。
 
+### 2026-03-17
+- 修复 qwen3.5 position_ids 创建错误
+- cli 参数增加 
+  - gguf_path: 本地 GGUF 模型权重路径（加载 GGUF 模型时需要）
+  - mmproj_path: 本地 mmproj GGUF 权重路径（加载多模态 GGUF 时需要）
+- WhichModel 增加 qwen3.5-gguf
+
 ### 2026-03-16
 - 增加 Qwen3.5 mmproj
 

docs/cli.md

@@ -18,6 +18,8 @@ aha [COMMAND] [OPTIONS]
 | `--weight-path <WEIGHT_PATH>` | Local model weight path | - |
 | `--save-dir <SAVE_DIR>` | Model download save directory | ~/.aha/ |
 | `--download-retries <DOWNLOAD_RETRIES>` | Download retry count | 3 |
+| `--gguf-path <GGUF_PATH>` | Local GGUF weight | - |
+| `--mmproj-path <MMPROJ_PATH>` | Local mmproj GGUF weight | - |
 | `-h, --help` | Display help information | - |
 | `-V, --version` | Display version number | - |
 
@@ -42,6 +44,8 @@ aha cli [OPTIONS] --model <MODEL>
 | `--weight-path <WEIGHT_PATH>` | Local model weight path (skip download if specified) | - |
 | `--save-dir <SAVE_DIR>` | Model download save directory | ~/.aha/ |
 | `--download-retries <DOWNLOAD_RETRIES>` | Download retry count | 3 |
+| `--gguf-path <GGUF_PATH>` | Local GGUF weight | - |
+| `--mmproj-path <MMPROJ_PATH>` | Local mmproj GGUF weight | - |
 
 **Examples:**
 
@@ -57,6 +61,9 @@ aha cli -m qwen3vl-2b --weight-path /path/to/model
 
 # Backward compatible way (equivalent to cli subcommand)
 aha -m qwen3vl-2b
+
+# use gguf-path and mmproj-path
+aha cli -m qwen3.5-gguf --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf
 ```
 
 ### run - Direct model inference
@@ -65,7 +72,7 @@ Run model inference directly without starting an HTTP service. Suitable for one-
 
 **Syntax:**
 ```bash
-aha run [OPTIONS] --model <MODEL> --input <INPUT> [--input <INPUT2>] --weight-path <WEIGHT_PATH>
+aha run [OPTIONS] --model <MODEL> --input <INPUT> [--input <INPUT2>] [--weight-path <WEIGHT_PATH>] [--gguf-path <GGUF_PATH>] [--mmproj-path <MMPROJ_PATH>]
 ```
 
 **Options:**
@@ -75,8 +82,9 @@ aha run [OPTIONS] --model <MODEL> --input <INPUT> [--input <INPUT2>] --weight-pa
 | `-m, --model <MODEL>` | Model type (required) | - |
 | `-i, --input <INPUT>` | Input text or file path (model-specific interpretation, supports 1-2 parameters: input1: prompt text, input2: file path) | - |
 | `-o, --output <OUTPUT>` | Output file path (optional, auto-generated if not specified) | - |
-| `--weight-path <WEIGHT_PATH>` | Local model weight path (required) | - |
+| `--weight-path <WEIGHT_PATH>` | Local model weight path (required when using non-GGUF models) | - |
-
+| `--gguf-path <GGUF_PATH>` | Local GGUF model weight path（required when using GGUF models） | - |
+| `--mmproj-path <MMPROJ_PATH>` | Local mmproj GGUF weight path（optional，If not specified, the module will not be loaded） | - |
 **Examples:**
 
 ```bash
@@ -109,6 +117,14 @@ aha run -m qwen2.5vl-3b -i "请分析图片并提取所有可见文本内容，
 
 # Qwen3-ASR speech recognition (single input: audio file)
 aha run -m qwen3asr-0.6b -i "audio.wav" --weight-path /path/to/model
+
+# Qwen3.5-GGUF without mmproj (single input: prompt text)
+aha run -m qwen3.5-gguf -i 你如何看待AI --gguf-path /path/to/xxx.gguf
+
+# Qwen3.5-GGUF with mmproj (two inputs：prompt text + file)
+aha run -m qwen3.5-gguf -i 提取图片中的文本 -i https://ai.bdstatic.com/file/C56CC9B274CF460CA33
+63E59ECD94423 --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf
+
 ```
 
 ### serv - Start service
@@ -117,7 +133,7 @@ Start HTTP service with a model. The `--weight-path` is optional - if not specif
 
 **Syntax:**
 ```bash
-aha serv [OPTIONS] --model <MODEL> [--weight-path <WEIGHT_PATH>]
+aha serv [OPTIONS] --model <MODEL> [--weight-path <WEIGHT_PATH>] [--gguf-path <GGUF_PATH>] [--mmproj-path <MMPROJ_PATH>]
 ```
 
 **Options:**
@@ -129,6 +145,8 @@ aha serv [OPTIONS] --model <MODEL> [--weight-path <WEIGHT_PATH>]
 | `-m, --model <MODEL>` | Model type (required) | - |
 | `--weight-path <WEIGHT_PATH>` | Local model weight path (optional) | ~/.aha/{model_id} |
 | `--allow-remote-shutdown` | Allow remote shutdown requests (not recommended) | false |
+| `--gguf-path <GGUF_PATH>` | Local GGUF model weight path（required when using GGUF models） | - |
+| `--mmproj-path <MMPROJ_PATH>` | Local mmproj GGUF weight path（optional，If not specified, the module will not be loaded） | - |
 
 **Examples:**
 

docs/cli.zh-CN.md

@@ -18,6 +18,8 @@ aha [COMMAND] [OPTIONS]
 | `--weight-path <WEIGHT_PATH>` | 本地模型权重路径 | - |
 | `--save-dir <SAVE_DIR>` | 模型下载保存目录 | ~/.aha/ |
 | `--download-retries <DOWNLOAD_RETRIES>` | 下载重试次数 | 3 |
+| `--gguf-path <GGUF_PATH>` | 本地 GGUF 模型权重 | - |
+| `--mmproj-path <MMPROJ_PATH>` | 本地 mmproj GGUF 模型权重 | - |
 | `-h, --help` | 显示帮助信息 | - |
 | `-V, --version` | 显示版本号 | - |
 
@@ -42,6 +44,8 @@ aha cli [OPTIONS] --model <MODEL>
 | `--weight-path <WEIGHT_PATH>` | 本地模型权重路径（如指定则跳过下载） | - |
 | `--save-dir <SAVE_DIR>` | 模型下载保存目录 | ~/.aha/ |
 | `--download-retries <DOWNLOAD_RETRIES>` | 下载重试次数 | 3 |
+| `--gguf-path <GGUF_PATH>` | 本地 GGUF 模型权重 | - |
+| `--mmproj-path <MMPROJ_PATH>` | 本地 mmproj GGUF 模型权重 | - |
 
 **示例：**
 
@@ -57,6 +61,9 @@ aha cli -m qwen3vl-2b --weight-path /path/to/model
 
 # 向后兼容方式（等同于 cli 子命令）
 aha -m qwen3vl-2b
+
+# 指定gguf-path和mmproj-path
+aha cli -m qwen3.5-gguf --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf
 ```
 
 ### run - 直接模型推理
@@ -65,7 +72,7 @@ aha -m qwen3vl-2b
 
 **语法：**
 ```bash
-aha run [OPTIONS] --model <MODEL> --input <INPUT> [--input <INPUT2>] --weight-path <WEIGHT_PATH>
+aha run [OPTIONS] --model <MODEL> --input <INPUT> [--input <INPUT2>] [--weight-path <WEIGHT_PATH>] [--gguf-path <GGUF_PATH>] [--mmproj-path <MMPROJ_PATH>]
 ```
 
 **选项：**
@@ -75,7 +82,9 @@ aha run [OPTIONS] --model <MODEL> --input <INPUT> [--input <INPUT2>] --weight-pa
 | `-m, --model <MODEL>` | 模型类型（必选） | - |
 | `-i, --input <INPUT>` | 输入文本或文件路径（模型特定解释，支持1-2个参数, input1： 提示文本, input2: 文件地址） | - |
 | `-o, --output <OUTPUT>` | 输出文件路径（可选，未指定则自动生成） | - |
-| `--weight-path <WEIGHT_PATH>` | 本地模型权重路径（必选） | - |
+| `--weight-path <WEIGHT_PATH>` | 本地模型权重路径（使用非GGUF模型时必选） | - |
+| `--gguf-path <GGUF_PATH>` | 本地GGUF模型权重路径（使用GGUF模型时必选） | - |
+| `--mmproj-path <MMPROJ_PATH>` | 本地mmproj GGUF模型权重路径（可选，未指定则不加载该模块） | - |
 
 **示例：**
 
@@ -109,6 +118,13 @@ aha run -m qwen2.5vl-3b -i "请分析图片并提取所有可见文本内容，
 
 # Qwen3-ASR 语音识别（单个输入：音频文件）
 aha run -m qwen3asr-0.6b -i "audio.wav" --weight-path /path/to/model
+
+# Qwen3.5-GGUF 无mmproj (单个输入：提示文本)
+aha run -m qwen3.5-gguf -i 你如何看待AI --gguf-path /path/to/xxx.gguf
+
+# Qwen3.5-GGUF 有mmproj (两个输入：提示文本 + 文件)
+aha run -m qwen3.5-gguf -i 提取图片中的文本 -i https://ai.bdstatic.com/file/C56CC9B274CF460CA33
+63E59ECD94423 --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf
 ```
 
 ### serv - 启动服务
@@ -117,7 +133,7 @@ aha run -m qwen3asr-0.6b -i "audio.wav" --weight-path /path/to/model
 
 **语法：**
 ```bash
-aha serv [OPTIONS] --model <MODEL> [--weight-path <WEIGHT_PATH>]
+aha serv [OPTIONS] --model <MODEL> [--weight-path <WEIGHT_PATH>] [--gguf-path <GGUF_PATH>] [--mmproj-path <MMPROJ_PATH>]
 ```
 
 **选项：**
@@ -129,6 +145,8 @@ aha serv [OPTIONS] --model <MODEL> [--weight-path <WEIGHT_PATH>]
 | `-m, --model <MODEL>` | 模型类型（必选） | - |
 | `--weight-path <WEIGHT_PATH>` | 本地模型权重路径（可选） | ~/.aha/{model_id} |
 | `--allow-remote-shutdown` | 允许远程关机请求（不推荐） | false |
+| `--gguf-path <GGUF_PATH>` | 本地GGUF模型权重路径（使用GGUF模型时必选） | - |
+| `--mmproj-path <MMPROJ_PATH>` | 本地mmproj GGUF模型权重路径（可选，未指定则不加载该模块） | - |
 
 **示例：**
 

src/api/mod.rs

@@ -37,9 +37,16 @@ static SHUTDOWN_FLAG: OnceLock<Arc<AtomicBool>> = OnceLock::new();
 static SERVER_PORT: OnceLock<u16> = OnceLock::new();
 static ALLOW_REMOTE_SHUTDOWN: OnceLock<bool> = OnceLock::new();
 
-pub fn init(model_type: WhichModel, path: String) -> anyhow::Result<()> {
+pub fn init(
+    model_type: WhichModel,
+    path: String,
+    gguf: Option<String>,
+    mmproj: Option<String>,
+) -> anyhow::Result<()> {
     let model_path = string_to_static_str(path);
-    let model = load_model(model_type, model_path)?;
+    let gguf = gguf.map(string_to_static_str);
+    let mmproj = mmproj.map(string_to_static_str);
+    let model = load_model(model_type, model_path, gguf, mmproj)?;
     MODEL.get_or_init(|| {
         Arc::new(RwLock::new(StoredModel {
             which_model: model_type,
@@ -247,6 +254,7 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str {
         WhichModel::Qwen3_5_2B => "qwen3.5-2b",
         WhichModel::Qwen3_5_4B => "qwen3.5-4b",
         WhichModel::Qwen3_5_9B => "qwen3.5-9b",
+        WhichModel::Qwen3_5Gguf => "qwen3.5-gguf",
         WhichModel::Qwen3ASR0_6B => "qwen3asr-0.6b",
         WhichModel::Qwen3ASR1_7B => "qwen3asr-1.7b",
         WhichModel::Qwen3vl2B => "qwen3vl-2b",
@@ -274,7 +282,8 @@ fn which_model_to_owner(which_model: WhichModel) -> &'static str {
         WhichModel::Qwen3vl2B
         | WhichModel::Qwen3vl4B
         | WhichModel::Qwen3vl8B
-        | WhichModel::Qwen3vl32B => "Qwen",
+        | WhichModel::Qwen3vl32B
+        | WhichModel::Qwen3_5Gguf => "Qwen",
         WhichModel::Qwen3_5_0_8B
         | WhichModel::Qwen3_5_2B
         | WhichModel::Qwen3_5_4B

src/exec/qwen3_5.rs

@@ -2,15 +2,138 @@
 
 use std::time::Instant;
 
-use anyhow::Result;
+use anyhow::{Result, anyhow};
 
 use crate::exec::ExecModel;
 use crate::models::GenerateModel;
 use crate::models::qwen3_5::generate::Qwen3_5GenerateModel;
-use crate::utils::get_file_path;
+use crate::utils::{get_file_path, string_to_static_str};
 
 pub struct Qwen3_5Exec;
 
+impl Qwen3_5Exec {
+    pub fn run_gguf(
+        input: &[String],
+        output: Option<&str>,
+        gguf_path: Option<String>,
+        mmproj_path: Option<String>,
+    ) -> Result<()> {
+        let input_text = &input[0];
+        let target_text = if input_text.starts_with("file://") {
+            let path = get_file_path(input_text)?;
+            std::fs::read_to_string(path)?
+        } else {
+            input_text.clone()
+        };
+        let model_file = if let Some(g) = gguf_path {
+            g
+        } else {
+            return Err(anyhow!("gguf model path is required"));
+        };
+        let mmproj_path = mmproj_path.map(string_to_static_str);
+
+        let i_start = Instant::now();
+        let mut model = Qwen3_5GenerateModel::init_from_gguf(&model_file, mmproj_path, None)?;
+        let i_duration = i_start.elapsed();
+        println!("Time elapsed in load model is: {:?}", i_duration);
+        let url = input.get(1);
+        let input_url = if let Some(url) = url
+            && (url.starts_with("http://")
+                || url.starts_with("https://")
+                || url.starts_with("file://"))
+        {
+            Some(url.clone())
+        } else {
+            url.map(|url| format!("file://{}", url))
+        };
+        let message = if let Some(input_url) = &input_url
+            && input_url.ends_with("mp4")
+        {
+            format!(
+                r#"{{
+            "model": "qwen3.5",
+            "messages": [
+                {{
+                    "role": "user",
+                    "content": [
+                        {{
+                            "type": "video",
+                            "video_url": 
+                            {{
+                                "url": "{}"
+                            }}
+                        }},
+                        {{
+                            "type": "text", 
+                            "text": "{}"
+                        }}
+                    ]
+                }}
+            ]
+        }}"#,
+                input_url, target_text
+            )
+        } else if let Some(input_url) = &input_url {
+            format!(
+                r#"{{
+            "model": "qwen3.5",
+            "messages": [
+                {{
+                    "role": "user",
+                    "content": [
+                        {{
+                            "type": "image",
+                            "image_url": {{
+                                "url": "{}"
+                            }}
+                        }},
+                        {{
+                            "type": "text", 
+                            "text": "{}"
+                        }}
+                    ]
+                }}
+            ]
+        }}"#,
+                input_url, target_text
+            )
+        } else {
+            format!(
+                r#"{{
+            "model": "qwen3.5",
+            "messages": [
+                {{
+                    "role": "user",
+                    "content": [
+                        {{
+                            "type": "text", 
+                            "text": "{}"
+                        }}
+                    ]
+                }}
+            ]
+        }}"#,
+                target_text
+            )
+        };
+        let mes = serde_json::from_str(&message)?;
+
+        let i_start = Instant::now();
+        let result = model.generate(mes)?;
+        let i_duration = i_start.elapsed();
+        println!("Time elapsed in generate is: {:?}", i_duration);
+
+        println!("Result: {:?}", result);
+
+        if let Some(out) = output {
+            std::fs::write(out, format!("{:?}", result))?;
+            println!("Output saved to: {}", out);
+        }
+
+        Ok(())
+    }
+}
+
 impl ExecModel for Qwen3_5Exec {
     fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
         let input_text = &input[0];
@@ -62,7 +185,7 @@ impl ExecModel for Qwen3_5Exec {
         } else {
             format!(
                 r#"{{
-            "model": "qwen2.5",
+            "model": "qwen3.5",
             "messages": [
                 {{
                     "role": "user",

src/exec/qwen3vl.rs

@@ -24,16 +24,19 @@ impl ExecModel for Qwen3vlExec {
         let mut model = Qwen3VLGenerateModel::init(weight_path, None, None)?;
         let i_duration = i_start.elapsed();
         println!("Time elapsed in load model is: {:?}", i_duration);
-        let url = &input[1];
+        let url = input.get(1);
-        let input_url = if url.starts_with("http://")
+        let input_url = if let Some(url) = url
-            || url.starts_with("https://")
+            && (url.starts_with("http://")
-            || url.starts_with("file://")
+                || url.starts_with("https://")
+                || url.starts_with("file://"))
         {
-            url.clone()
+            Some(url.clone())
         } else {
-            format!("file://{}", url)
+            url.map(|url| format!("file://{}", url))
         };
-        let message = if input_url.ends_with("mp4") {
+        let message = if let Some(input_url) = &input_url
+            && input_url.ends_with("mp4")
+        {
             format!(
                 r#"{{
             "model": "qwen3vl",
@@ -58,7 +61,7 @@ impl ExecModel for Qwen3vlExec {
         }}"#,
                 input_url, target_text
             )
-        } else {
+        } else if let Some(input_url) = &input_url {
             format!(
                 r#"{{
             "model": "qwen3vl",
@@ -82,6 +85,24 @@ impl ExecModel for Qwen3vlExec {
         }}"#,
                 input_url, target_text
             )
+        } else {
+            format!(
+                r#"{{
+            "model": "qwen3vl",
+            "messages": [
+                {{
+                    "role": "user",
+                    "content": [
+                        {{
+                            "type": "text", 
+                            "text": "{}"
+                        }}
+                    ]
+                }}
+            ]
+        }}"#,
+                target_text
+            )
         };
         let mes = serde_json::from_str(&message)?;
 

src/main.rs

@@ -6,6 +6,7 @@ use aha::{
     process::{cleanup_pid_file, create_pid_file},
     utils::{download_model, get_default_save_dir},
 };
+use anyhow::anyhow;
 use clap::{Args, Parser, Subcommand, ValueEnum};
 use rocket::{
     Config,
@@ -45,6 +46,14 @@ struct Cli {
     #[arg(long)]
     download_retries: Option<u32>,
 
+    /// Local GGUF model weight path (required for loading models with GGUF).
+    #[arg(long)]
+    gguf_path: Option<String>,
+
+    /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF)
+    #[arg(long)]
+    mmproj_path: Option<String>,
+
     #[command(subcommand)]
     command: Option<Commands>,
 }
@@ -104,6 +113,14 @@ struct CliArgs {
     /// Download retry count
     #[arg(long)]
     download_retries: Option<u32>,
+
+    /// Local GGUF model weight path (required for loading models with GGUF).
+    #[arg(long)]
+    gguf_path: Option<String>,
+
+    /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF)
+    #[arg(long)]
+    mmproj_path: Option<String>,
 }
 
 /// Arguments for the 'serv start' subcommand
@@ -115,6 +132,14 @@ struct ServArgs {
     /// Local model weight path (defaults to ~/.aha/{model_id} if not specified)
     #[arg(long)]
     weight_path: Option<String>,
+
+    /// Local GGUF model weight path (required for loading models with GGUF).
+    #[arg(long)]
+    gguf_path: Option<String>,
+
+    /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF)
+    #[arg(long)]
+    mmproj_path: Option<String>,
 }
 
 /// Arguments for the 'serv list' subcommand
@@ -159,6 +184,14 @@ struct RunArgs {
     /// Local model weight path (defaults to ~/.aha/{model_id} if not specified)
     #[arg(long)]
     weight_path: Option<String>,
+
+    /// Local GGUF model weight path (required for loading models with GGUF).
+    #[arg(long)]
+    gguf_path: Option<String>,
+
+    /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF)
+    #[arg(long)]
+    mmproj_path: Option<String>,
 }
 
 /// Arguments for the 'delete' subcommand (delete model from default location)
@@ -282,23 +315,33 @@ async fn run_cli(args: CliArgs) -> anyhow::Result<()> {
         weight_path,
         save_dir,
         download_retries,
+        gguf_path,
+        mmproj_path,
     } = args;
     let model_id = common.model.model_id();
 
-    let model_path = match weight_path {
+    let (model_path, gguf, mmproj) = if model_id.eq("GGUF") {
-        Some(path) => path,
+        if gguf_path.is_none() {
-        None => {
+            return Err(anyhow!("gguf model path is required"));
-            let save_dir = match save_dir {
-                Some(dir) => dir,
-                None => get_default_save_dir().expect("Failed to get home directory"),
-            };
-            let max_retries = download_retries.unwrap_or(3);
-            download_model(model_id, &save_dir, max_retries).await?;
-            save_dir + "/" + model_id
         }
+        ("GGUF".to_string(), gguf_path, mmproj_path)
+    } else {
+        let model_path = match weight_path {
+            Some(path) => path,
+            None => {
+                let save_dir = match save_dir {
+                    Some(dir) => dir,
+                    None => get_default_save_dir().expect("Failed to get home directory"),
+                };
+                let max_retries = download_retries.unwrap_or(3);
+                download_model(model_id, &save_dir, max_retries).await?;
+                save_dir + "/" + model_id
+            }
+        };
+        (model_path, None, None)
     };
 
-    init(common.model, model_path)?;
+    init(common.model, model_path, gguf, mmproj)?;
     start_http_server(common.address, common.port, common.allow_remote_shutdown).await?;
 
     Ok(())
@@ -309,14 +352,24 @@ async fn run_serv(args: ServArgs) -> anyhow::Result<()> {
     let ServArgs {
         common,
         weight_path,
+        gguf_path,
+        mmproj_path,
     } = args;
-
+    let model_id = common.model.model_id();
-    let model_path = match weight_path {
+    let (model_path, gguf, mmproj) = if model_id.eq("GGUF") {
-        Some(path) => path,
+        if gguf_path.is_none() {
-        None => get_default_weight_path(common.model),
+            return Err(anyhow!("gguf model path is required"));
+        }
+        ("GGUF".to_string(), gguf_path, mmproj_path)
+    } else {
+        let model_path = match weight_path {
+            Some(path) => path,
+            None => get_default_weight_path(common.model),
+        };
+        (model_path, None, None)
     };
 
-    init(common.model, model_path)?;
+    init(common.model, model_path, gguf, mmproj)?;
     start_http_server(common.address, common.port, common.allow_remote_shutdown).await?;
 
     Ok(())
@@ -392,6 +445,8 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
         input,
         output,
         weight_path,
+        gguf_path,
+        mmproj_path,
     } = args;
 
     // Use default weight path if not specified
@@ -399,7 +454,6 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
         Some(path) => path,
         None => get_default_weight_path(model),
     };
-
     match model {
         WhichModel::MiniCPM4_0_5B => {
             use aha::exec::minicpm4::MiniCPM4Exec;
@@ -433,6 +487,10 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
             use aha::exec::qwen3_5::Qwen3_5Exec;
             Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
         }
+        WhichModel::Qwen3_5Gguf => {
+            use aha::exec::qwen3_5::Qwen3_5Exec;
+            Qwen3_5Exec::run_gguf(&input, output.as_deref(), gguf_path, mmproj_path)?;
+        }
         WhichModel::Qwen3ASR0_6B => {
             use aha::exec::qwen3_asr::Qwen3ASRExec;
             Qwen3ASRExec::run(&input, output.as_deref(), &weight_path)?;
@@ -610,6 +668,8 @@ async fn main() -> anyhow::Result<()> {
                 weight_path: cli.weight_path,
                 save_dir: cli.save_dir,
                 download_retries: cli.download_retries,
+                gguf_path: cli.gguf_path,
+                mmproj_path: cli.mmproj_path,
             };
             run_cli(args).await
         }

src/models/common/gguf.rs

@@ -213,6 +213,10 @@ impl QuantizedLinear {
     pub fn new(inner: QMatMul, bias: Option<Tensor>) -> Self {
         Self { inner, bias }
     }
+
+    pub fn inner_dequantize(&self) -> Result<Tensor> {
+        Ok(self.inner.dequantize_f16()?)
+    }
 }
 
 impl Module for QuantizedLinear {

src/models/mod.rs

@@ -22,7 +22,7 @@ pub mod w2v_bert_2_0;
 use aha_openai_dive::v1::resources::chat::{
     ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse,
 };
-use anyhow::Result;
+use anyhow::{Result, anyhow};
 use rocket::futures::Stream;
 
 use crate::models::{
@@ -54,6 +54,8 @@ pub enum WhichModel {
     Qwen3_5_4B,
     #[value(name = "qwen3.5-9b", hide = true)]
     Qwen3_5_9B,
+    #[value(name = "qwen3.5-gguf", hide = true)]
+    Qwen3_5Gguf,
     #[value(name = "qwen3asr-0.6b", hide = true)]
     Qwen3ASR0_6B,
     #[value(name = "qwen3asr-1.7b", hide = true)]
@@ -98,6 +100,7 @@ impl WhichModel {
             WhichModel::Qwen3_5_2B => "Qwen/Qwen3.5-2B",
             WhichModel::Qwen3_5_4B => "Qwen/Qwen3.5-4B",
             WhichModel::Qwen3_5_9B => "Qwen/Qwen3.5-9B",
+            WhichModel::Qwen3_5Gguf => "GGUF",
             WhichModel::Qwen3ASR0_6B => "Qwen/Qwen3-ASR-0.6B",
             WhichModel::Qwen3ASR1_7B => "Qwen/Qwen3-ASR-1.7B",
             WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct",
@@ -130,7 +133,8 @@ impl WhichModel {
             | WhichModel::Qwen3_5_0_8B
             | WhichModel::Qwen3_5_2B
             | WhichModel::Qwen3_5_4B
-            | WhichModel::Qwen3_5_9B => "vlm",
+            | WhichModel::Qwen3_5_9B
+            | WhichModel::Qwen3_5Gguf => "vlm",
             // OCR models
             WhichModel::DeepSeekOCR
             | WhichModel::HunyuanOCR
@@ -229,7 +233,12 @@ impl<'a> GenerateModel for ModelInstance<'a> {
     }
 }
 
-pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_>> {
+pub fn load_model<'a>(
+    model_type: WhichModel,
+    path: &str,
+    gguf: Option<&str>,
+    mmproj: Option<&str>,
+) -> Result<ModelInstance<'a>> {
     let model = match model_type {
         WhichModel::MiniCPM4_0_5B => {
             let model = MiniCPMGenerateModel::init(path, None, None)?;
@@ -263,6 +272,14 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_
             let model = Qwen3_5GenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3_5(model)
         }
+        WhichModel::Qwen3_5Gguf => {
+            if gguf.is_none() {
+                return Err(anyhow!("Qwen3_5Gguf gguf model path is required"));
+            }
+            let gguf = gguf.unwrap();
+            let model = Qwen3_5GenerateModel::init_from_gguf(gguf, mmproj, None)?;
+            ModelInstance::Qwen3_5(model)
+        }
         WhichModel::Qwen3ASR0_6B => {
             let model = Qwen3AsrGenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3ASR(model)

src/models/qwen3_5/generate.rs

@@ -102,7 +102,9 @@ impl<'a> Qwen3_5GenerateModel<'a> {
             (None, None)
         };
 
-        // let eos_token_id = gguf.get_matedata("tokenizer.ggml.eos_token_id")?.to_u32()?;
+        let eos_token_id = model_gguf
+            .get_matedata("tokenizer.ggml.eos_token_id")?
+            .to_u32()?;
         let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut model_gguf, mmproj_gguf.as_mut(), &device)?;
         let stem = std::path::Path::new(model_file)
             .file_stem() // 获取文件名主干（不含扩展名）
@@ -114,7 +116,8 @@ impl<'a> Qwen3_5GenerateModel<'a> {
             pre_processor,
             qwen3_5,
             device,
-            eos_token_id: 248044,
+            // eos_token_id: 248044,
+            eos_token_id,
             model_name: stem.to_string(),
             repeat_penalty: 1.1,
             repeat_last_n: 64,
@@ -125,7 +128,7 @@ impl<'a> Qwen3_5GenerateModel<'a> {
 impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
     fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
         let seed = mes.seed.unwrap_or(32768) as u64;
-        let temperature = mes.temperature.unwrap_or(0.6);
+        let temperature = mes.temperature.unwrap_or(0.4);
         let top_p = mes.top_p.unwrap_or(0.95);
         let mut logit_processor =
             get_logit_processor(temperature.into(), top_p.into(), Some(20), seed);
@@ -144,7 +147,6 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
             } else {
                 (mes_render, None, None, None, None)
             };
-        // let input = self.pre_processor.process_info(&mes, &mes_render)?;
         let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?;
         let mut seq_len = input_ids.dim(1)?;
         let prompt_tokens = seq_len as u32;

src/models/qwen3_5/model.rs

@@ -750,7 +750,7 @@ impl Qwen3_5Attention {
     }
 
     pub fn clear_kv_cache(&mut self) {
-        self.kv_cache = None
+        self.kv_cache = None;
     }
 }
 
@@ -1024,7 +1024,6 @@ impl Qwen3_5TextModel {
             // i += 1;
         }
         xs = self.norm.forward(&xs)?;
-        // println!("norm : {}", xs);
         Ok(xs)
     }
 
@@ -1348,7 +1347,9 @@ impl Qwen3_5Model {
         video_grid_thw: Option<&Tensor>,
         seqlen_offset: usize,
     ) -> Result<Tensor> {
-        let position_ids = if let Some(rope_deltas) = &self.rope_deltas {
+        let position_ids = if let Some(rope_deltas) = &self.rope_deltas
+            && seqlen_offset != 0
+        {
             let (bs, seq_len, _) = inputs_embeds.dims3()?;
             Tensor::arange(
                 seqlen_offset as i64,
@@ -1383,12 +1384,12 @@ impl Qwen3_5Model {
         seqlen_offset: usize,
     ) -> Result<Tensor> {
         let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?;
-        // println!("embed_tokens: {}", inputs_embeds);
         if let Some(pixel_values) = pixel_values
             && let Some(image_grid_thw) = image_grid_thw
             && let Some(visual) = self.visual.as_ref()
         {
             let (image_embeds, _) = visual.forward(pixel_values, image_grid_thw)?;
+            // println!("image_embeds: {}", image_embeds);
             let vision_mask = get_equal_mask(input_ids, self.image_token_id)?;
             let n_image_tokens = vision_mask.sum_all()?.to_scalar::<u32>()?;
             if n_image_tokens as usize != image_embeds.dim(0)? {
@@ -1429,9 +1430,7 @@ impl Qwen3_5Model {
         let outputs = self.language_model.forward(&inputs_embeds, &position_ids)?;
         let seq_len = outputs.dim(1)?;
         let hidden_state = outputs.narrow(1, seq_len - 1, 1)?;
-        // println!("narrow 1 : {}", hidden_state);
         let logits = self.lm_head.forward(&hidden_state)?;
-        // println!("logits : {}", logits);
         Ok(logits)
     }
 

src/models/qwen3vl/model.rs

@@ -95,8 +95,9 @@ impl Qwen3VLVisionPatchEmbed {
     pub fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
         // hidden_states shape:  (grid_t*grid_h*grid_w, c*temporal_patch_size*patch_size*patch_size)
         // ((), 1536) matmul (1536, 1024) -> ((), 1024)
-        let hidden_states = hidden_states.matmul(&self.conv3d_weight)?;
+        let dtype = hidden_states.dtype();
-        let hidden_states = hidden_states.broadcast_add(&self.conv3d_bias)?;
+        let hidden_states = hidden_states.matmul(&self.conv3d_weight.to_dtype(dtype)?)?;
+        let hidden_states = hidden_states.broadcast_add(&self.conv3d_bias.to_dtype(dtype)?)?;
         Ok(hidden_states)
     }
 }
@@ -169,7 +170,12 @@ impl Qwen3VLVisionPatchMerger {
         } else {
             xs.clone()
         };
-        let xs = self.norm.forward(&xs)?.reshape(((), self.hidden_size))?;
+        let orig_dtype = xs.dtype();
+        let xs = self
+            .norm
+            .forward(&xs.to_dtype(self.norm.weight().dtype())?)?
+            .reshape(((), self.hidden_size))?;
+        let xs = xs.to_dtype(orig_dtype)?;
         let xs = self
             .linear_fc2
             .forward(&self.act_fn.forward(&self.linear_fc1.forward(&xs)?)?)?;
@@ -343,12 +349,21 @@ impl Qwen3VLVisionBlock {
         cos: &Tensor,
         sin: &Tensor,
     ) -> Result<Tensor> {
+        let orig_dtype = xs.dtype();
         let residual = xs.clone();
-        let xs = self.norm1.forward(xs)?;
+        let xs = self
+            .norm1
+            .forward(&xs.to_dtype(self.norm1.weight().dtype())?)?;
+        let xs = xs.to_dtype(orig_dtype)?;
         let xs = self.attn.forward(&xs, cos, sin, cu_seqlens)?;
         let xs = (residual + xs)?;
         let residual = xs.clone();
-        let xs = self.mlp.forward(&self.norm2.forward(&xs)?)?;
+        let xs = self.mlp.forward(
+            &self
+                .norm2
+                .forward(&xs.to_dtype(self.norm2.weight().dtype())?)?
+                .to_dtype(orig_dtype)?,
+        )?;
         let xs = (residual + xs)?;
         Ok(xs)
     }
@@ -679,7 +694,9 @@ impl Qwen3VLVisionModel {
         grid_thw: &Tensor,
     ) -> Result<(Tensor, Vec<Tensor>)> {
         let hidden_states = self.patch_embed.forward(hidden_states)?;
-        let pos_embeds = self.fast_pos_embed_interpolate(grid_thw)?;
+        let pos_embeds = self
+            .fast_pos_embed_interpolate(grid_thw)?
+            .to_dtype(hidden_states.dtype())?;
         let hidden_states = hidden_states.broadcast_add(&pos_embeds)?;
         let rotary_pos_emb = self.rot_pos_emb(grid_thw)?;
         let seq_len = hidden_states.dim(0)?;

src/position_embed/rope.rs

@@ -520,6 +520,7 @@ impl Qwen3VLTextRotaryEmbedding {
         let position_ids_expanded = position_ids
             .unsqueeze(D::Minus2)?
             .to_dtype(DType::F32)?
+            // .to_dtype(dtype)?
             .contiguous()?;
         // inv_freq Vec<f32> -> Tensor(1, 1, head_dim / 2, 1) -> (3, bs, head_dim / 2, 1)
         let inv_freq_expanded = Tensor::from_vec(
@@ -529,6 +530,7 @@ impl Qwen3VLTextRotaryEmbedding {
         )?
         .broadcast_as((3, position_ids.dim(1)?, self.inv_freq.len(), 1))?
         .to_dtype(DType::F32)?
+        // .to_dtype(dtype)?
         .contiguous()?;
 
         // (3, bs, head_dim / 2, 1) matmul (3, bs, 1, position)

tests/messy_test.rs

@@ -10,12 +10,32 @@
 use anyhow::Result;
 // use byteorder::{LittleEndian, ReadBytesExt};
 use candle_core::Tensor;
+use modelscope::{DownloadOptions, ModelScope};
 // use sentencepiece::SentencePieceProcessor;
 // use zip::ZipArchive;
 
+#[tokio::test]
+async fn download_test() -> Result<()> {
+    // cargo test -F cuda --test messy_test download_test -r -- --nocapture
+    let model_id = "unsloth/Qwen3.5-4B-GGUF";
+    let model_name = "Qwen3.5-4B-IQ4_NL.gguf";
+    let save_dir =
+        aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
+    let _ = ModelScope::download_with_options(
+        model_id,
+        save_dir,
+        DownloadOptions {
+            files: (vec![model_name.to_string()]).into(),
+        },
+    )
+    .await;
+    Ok(())
+}
+
 #[test]
 fn messy_test() -> Result<()> {
     // RUST_BACKTRACE=1 cargo test -F cuda --test messy_test messy_test -r -- --nocapture
+
     let device = &candle_core::Device::Cpu;
     let t1 = Tensor::randn(0.0, 1.0, (16, 9, 64, 128), device)?;
     let t2 = Tensor::randn(0.0, 1.0, (16, 9, 128, 64), device)?;

tests/test_gguf_qwen3_5.rs

@@ -9,13 +9,19 @@ use anyhow::Result;
 #[test]
 fn gguf_test() -> Result<()> {
     // RUST_BACKTRACE=1 cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture
-    // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题
+    // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q6_K.gguf"; // 有问题
     // let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/mmproj-F16.gguf";
     // let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf";
+    // let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/mmproj-F16.gguf";
     let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf";
     let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/mmproj-F16.gguf";
     // let mut model_file = std::fs::File::open(model_path)?;
     // let model = gguf_file::Content::read(&mut model_file)?;
+    // for (key, value) in model.tensor_infos {
+    //     if key.contains("blk.12.") {
+    //         println!("{key}: {:#?}", value);
+    //     }
+    // }
     // for (key, value) in model.metadata {
     //     if key.contains("tokeni") {
     //         continue;