From 44d91da650453b6b11f0598b9c1cea0b0a2e460c Mon Sep 17 00:00:00 2001 From: jhqxxx <18280426169@163.com> Date: Fri, 3 Apr 2026 20:43:51 +0800 Subject: [PATCH] CLI update: subcommand must be specified --- README.md | 5 +- README.zh-CN.md | 6 +- docs/changelog.md | 3 +- docs/changelog.zh-CN.md | 1 + docs/cli.md | 45 +++++------- docs/cli.zh-CN.md | 51 ++++++------- docs/installation.md | 3 +- docs/installation.zh-CN.md | 3 +- docs/supported-models.md | 5 +- docs/supported-models.zh-CN.md | 5 +- src/cli/args.rs | 113 ++++++++++------------------- src/cli/mod.rs | 86 +++++++++++++++------- src/main.rs | 35 ++------- src/models/common/model_mapping.rs | 27 ++++++- src/models/mod.rs | 39 +++++++--- src/models/qwen3_5/generate.rs | 4 +- src/server/api.rs | 21 +++++- 17 files changed, 240 insertions(+), 212 deletions(-) diff --git a/README.md b/README.md index 697379f..10455b4 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,9 @@ aha is a high-performance, cross-platform AI inference engine built with Rust an ## Changelog ### 2026-04-03 +- CLI update: subcommand must be specified - ChatCompletionParameters add repeat_penalty and repeat_last_n -- generate add penalty repeat code +- generate add penalty repeat ### 2026-04-02 - refactor generate code @@ -113,7 +114,7 @@ aha list aha download -m Qwen/Qwen3-ASR-0.6B # Download model and start service -aha -m Qwen/Qwen3-ASR-0.6B +aha cli -m Qwen/Qwen3-ASR-0.6B # Run inference directly (without starting service) aha run -m Qwen/Qwen3-ASR-0.6B -i "audio.wav" diff --git a/README.zh-CN.md b/README.zh-CN.md index ef33e2c..55673fa 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -47,6 +47,7 @@ aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理 ## 更新日志 ## Changelog ### 2026-04-03 +- CLI 更新: 必须指定子命令 - ChatCompletionParameters 新增 repeat_penalty 和 repeat_last_n 参数 - 生成添加重复惩罚代码 @@ -73,9 +74,6 @@ aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理 - 新增 LFM2.5-1.2B-Instruct - 新增 LFM2-1.2B -### v0.2.3 (2026-03-18) -- 新增 DeepSeek-OCR-2 - **[查看完整更新日志](docs/changelog.zh-CN.md)** → @@ -116,7 +114,7 @@ aha list aha download -m Qwen/Qwen3-ASR-0.6B # 下载模型并启动服务 -aha -m Qwen/Qwen3-ASR-0.6B +aha cli -m Qwen/Qwen3-ASR-0.6B # 直接运行推理（无需启动服务） aha run -m Qwen/Qwen3-ASR-0.6B -i "audio.wav" diff --git a/docs/changelog.md b/docs/changelog.md index 44f314a..b7e63e6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,8 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### 2026-04-03 +- CLI update: subcommand must be specified - ChatCompletionParameters add repeat_penalty and repeat_last_n -- generate add penalty repeat code +- generate add penalty repeat ### 2026-04-02 - refactor generate code diff --git a/docs/changelog.zh-CN.md b/docs/changelog.zh-CN.md index 84646f1..e695352 100644 --- a/docs/changelog.zh-CN.md +++ b/docs/changelog.zh-CN.md @@ -6,6 +6,7 @@ 本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/spec/v2.0.0.html)。 ### 2026-04-03 +- CLI 更新: 必须指定子命令 - ChatCompletionParameters 新增 repeat_penalty 和 repeat_last_n 参数 - 生成添加重复惩罚代码 diff --git a/docs/cli.md b/docs/cli.md index 29a6522..bf497a3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -18,16 +18,19 @@ aha [COMMAND] [OPTIONS] | `--weight-path ` | Local model weight path | - | | `--save-dir ` | Model download save directory | ~/.aha/ | | `--download-retries ` | Download retry count | 3 | -| `--gguf-path ` | Local GGUF weight | - | +| `--gguf-path ` | Local GGUF weight（required when using GGUF models） | - | | `--mmproj-path ` | Local mmproj GGUF weight | - | +| `--onnx-path ` | Local ONNX weight（required when using ONNX models） | - | +| `--config-path ` | extra config path for gguf/onnx | - | | `-h, --help` | Display help information | - | | `-V, --version` | Display version number | - | ## Commands -### cli - Download model and start service (default) +### cli - Download model and start service -Download the specified model and start an HTTP service. This command is used by default when no subcommand is specified. +Download the specified model and start an HTTP service. +Download only supports models in safetensors format; for GGUF/ONNX models, you must specify a local file path. **Syntax:** ```bash @@ -44,8 +47,10 @@ aha cli [OPTIONS] --model | `--weight-path ` | Local model weight path (skip download if specified) | - | | `--save-dir ` | Model download save directory | ~/.aha/ | | `--download-retries ` | Download retry count | 3 | -| `--gguf-path ` | Local GGUF weight | - | +| `--gguf-path ` | Local GGUF weight（required when using GGUF models） | - | | `--mmproj-path ` | Local mmproj GGUF weight | - | +| `--onnx-path ` | Local ONNX weight（required when using ONNX models） | - | +| `--config-path ` | extra config path for gguf/onnx | - | **Examples:** @@ -59,9 +64,6 @@ aha cli -m Qwen/Qwen3-VL-2B-Instruct -p 8080 --save-dir /data/models # Use local model (skip download) aha cli -m Qwen/Qwen3-VL-2B-Instruct --weight-path /path/to/model -# Backward compatible way (equivalent to cli subcommand) -aha -m Qwen/Qwen3-VL-2B-Instruct - # use gguf-path and mmproj-path aha cli -m qwen3.5-gguf --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf ``` @@ -72,7 +74,7 @@ Run model inference directly without starting an HTTP service. Suitable for one- **Syntax:** ```bash -aha run [OPTIONS] --model --input [--input ] [--weight-path ] [--gguf-path ] [--mmproj-path ] +aha run [OPTIONS] --model --input [--input ] [--weight-path ] [--gguf-path ] [--mmproj-path ] [--onnx-path ] [--config-path ] ``` **Options:** @@ -82,9 +84,12 @@ aha run [OPTIONS] --model --input [--input ] [--weight-p | `-m, --model ` | Model type (required) | - | | `-i, --input ` | Input text or file path (model-specific interpretation, supports 1-2 parameters: input1: prompt text, input2: file path) | - | | `-o, --output ` | Output file path (optional, auto-generated if not specified) | - | -| `--weight-path ` | Local model weight path (required when using non-GGUF models) | - | +| `--weight-path ` | Local model weight path (required when using safetensors models) | - | | `--gguf-path ` | Local GGUF model weight path（required when using GGUF models） | - | | `--mmproj-path ` | Local mmproj GGUF weight path（optional，If not specified, the module will not be loaded） | - | +| `--onnx-path ` | Local ONNX weight（required when using ONNX models） | - | +| `--config-path ` | extra config path for gguf/onnx | - | + **Examples:** ```bash @@ -129,11 +134,13 @@ aha run -m qwen3.5-gguf -i 提取图片中的文本 -i https://ai.bdstatic.com/f ### serv - Start service -Start HTTP service with a model. The `--weight-path` is optional - if not specified, it defaults to `~/.aha/{model_id}`. +Start HTTP service with a model. +Safetensors model: The `--weight-path` is optional - if not specified, it defaults to `~/.aha/{model_id}`. +GGUF/ONNX model: The `--gguf-path`/ `--onnx-path` must be specified **Syntax:** ```bash -aha serv [OPTIONS] --model [--weight-path ] [--gguf-path ] [--mmproj-path ] +aha serv [OPTIONS] --model [--weight-path ] [--gguf-path ] [--mmproj-path ] [--onnx-path ] [--config-path ] ``` **Options:** @@ -336,7 +343,7 @@ Example: ```bash # One command to download and start service -aha -m Qwen/Qwen3-VL-2B-Instruct +aha cli -m Qwen/Qwen3-VL-2B-Instruct ``` ### Scenario 2: Start service with existing model @@ -360,7 +367,7 @@ aha serv -m Qwen/Qwen3-VL-2B-Instruct --weight-path /data/models/Qwen/Qwen3-VL-2 ```bash # Start service on 0.0.0.0:8080, allow external access -aha -m Qwen/Qwen3-VL-2B-Instruct -a 0.0.0.0 -p 8080 +aha cli -m Qwen/Qwen3-VL-2B-Instruct -a 0.0.0.0 -p 8080 ``` ## API Endpoints @@ -395,18 +402,6 @@ After the service starts, the following API endpoints are available: - **Format**: JSON response -## Backward Compatibility - -To maintain compatibility with older versions, the following two usage methods are equivalent: - -```bash -# New way (recommended) -aha cli -m Qwen/Qwen3-VL-2B-Instruct - -# Old way (backward compatible) -aha -m Qwen/Qwen3-VL-2B-Instruct -``` - ## Notes 1. **serv subcommand requires `--weight-path`**: Since the `serv` subcommand does not download models, you must specify the path to an already downloaded model via `--weight-path`. diff --git a/docs/cli.zh-CN.md b/docs/cli.zh-CN.md index 78d495a..7a68976 100644 --- a/docs/cli.zh-CN.md +++ b/docs/cli.zh-CN.md @@ -15,19 +15,22 @@ aha [COMMAND] [OPTIONS] | `-a, --address
` | 服务监听地址 | 127.0.0.1 | | `-p, --port ` | 服务监听端口 | 10100 | | `-m, --model ` | 模型类型（必选） | - | -| `--weight-path ` | 本地模型权重路径 | - | +| `--weight-path ` | 本地safetensors模型权重路径 | - | | `--save-dir ` | 模型下载保存目录 | ~/.aha/ | | `--download-retries ` | 下载重试次数 | 3 | -| `--gguf-path ` | 本地 GGUF 模型权重 | - | -| `--mmproj-path ` | 本地 mmproj GGUF 模型权重 | - | +| `--gguf-path ` | 本地 GGUF 模型权重（使用GGUF模型时必选） | - | +| `--mmproj-path ` | 本地 mmproj GGUF 模型权重（可选，未指定则不加载该模块） | - | +| `--onnx-path ` | 本地 ONNX 模型权重 （使用ONNX模型时必选） | - | +| `--config-path ` | GGUF/ONNX 需要的额外配置路径(可选) | - | | `-h, --help` | 显示帮助信息 | - | | `-V, --version` | 显示版本号 | - | ## 子命令 -### cli - 下载模型并启动服务（默认） +### cli - 下载模型并启动服务 -下载指定的模型并启动 HTTP 服务。当不指定子命令时，默认使用此命令。 +下载指定的模型并启动 HTTP 服务。 +下载仅支持safetensors格式模型, GGUF/ONNX模型必须指定本地文件路径 **语法：** ```bash @@ -41,11 +44,13 @@ aha cli [OPTIONS] --model | `-a, --address
` | 服务监听地址 | 127.0.0.1 | | `-p, --port ` | 服务监听端口 | 10100 | | `-m, --model ` | 模型类型（必选） | - | -| `--weight-path ` | 本地模型权重路径（如指定则跳过下载） | - | +| `--weight-path ` | 本地safetensors模型权重路径（如指定则跳过下载） | - | | `--save-dir ` | 模型下载保存目录 | ~/.aha/ | | `--download-retries ` | 下载重试次数 | 3 | -| `--gguf-path ` | 本地 GGUF 模型权重 | - | -| `--mmproj-path ` | 本地 mmproj GGUF 模型权重 | - | +| `--gguf-path ` | 本地 GGUF 模型权重（使用GGUF模型时必选） | - | +| `--mmproj-path ` | 本地 mmproj GGUF 模型权重（可选，未指定则不加载该模块） | - | +| `--onnx-path ` | 本地 ONNX 模型权重 （使用ONNX模型时必选） | - | +| `--config-path ` | GGUF/ONNX 需要的额外配置路径(可选) | - | **示例：** @@ -59,9 +64,6 @@ aha cli -m Qwen/Qwen3-VL-2B-Instruct -p 8080 --save-dir /data/models # 使用本地模型（不下载） aha cli -m Qwen/Qwen3-VL-2B-Instruct --weight-path /path/to/model -# 向后兼容方式（等同于 cli 子命令） -aha -m Qwen/Qwen3-VL-2B-Instruct - # 指定gguf-path和mmproj-path aha cli -m qwen3.5-gguf --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmproj-xxx.gguf ``` @@ -72,7 +74,7 @@ aha cli -m qwen3.5-gguf --gguf-path /path/to/xxx.gguf --mmproj-path /path/to/mmp **语法：** ```bash -aha run [OPTIONS] --model --input [--input ] [--weight-path ] [--gguf-path ] [--mmproj-path ] +aha run [OPTIONS] --model --input [--input ] [--weight-path ] [--gguf-path ] [--mmproj-path ] [--onnx-path ] [--config-path ] ``` **选项：** @@ -82,9 +84,11 @@ aha run [OPTIONS] --model --input [--input ] [--weight-p | `-m, --model ` | 模型类型（必选） | - | | `-i, --input ` | 输入文本或文件路径（模型特定解释，支持1-2个参数, input1： 提示文本, input2: 文件地址） | - | | `-o, --output ` | 输出文件路径（可选，未指定则自动生成） | - | -| `--weight-path ` | 本地模型权重路径（使用非GGUF模型时必选） | - | +| `--weight-path ` | 本地模型权重路径（使用Safetensors模型时必选） | - | | `--gguf-path ` | 本地GGUF模型权重路径（使用GGUF模型时必选） | - | | `--mmproj-path ` | 本地mmproj GGUF模型权重路径（可选，未指定则不加载该模块） | - | +| `--onnx-path ` | 本地 ONNX 模型权重 （使用ONNX模型时必选） | - | +| `--config-path ` | GGUF/ONNX 需要的额外配置路径(可选) | - | **示例：** @@ -129,7 +133,9 @@ aha run -m qwen3.5-gguf -i 提取图片中的文本 -i https://ai.bdstatic.com/f ### serv - 启动服务 -使用指定模型启动 HTTP 服务。`--weight-path` 是可选的 - 如果不指定，默认使用 `~/.aha/{model_id}`。 +使用指定模型启动 HTTP 服务。 +safetensors模型`--weight-path` 是可选的 - 如果不指定，默认使用 `~/.aha/{model_id}`。 +GGUF/ONNX模型必须指定本地文件路径 **语法：** ```bash @@ -147,6 +153,8 @@ aha serv [OPTIONS] --model [--weight-path ] [--gguf-path ` | 本地GGUF模型权重路径（使用GGUF模型时必选） | - | | `--mmproj-path ` | 本地mmproj GGUF模型权重路径（可选，未指定则不加载该模块） | - | +| `--onnx-path ` | 本地 ONNX 模型权重 （使用ONNX模型时必选） | - | +| `--config-path ` | GGUF/ONNX 需要的额外配置路径(可选) | - | **示例：** @@ -336,7 +344,7 @@ aha list -j ```bash # 一条命令下载并启动服务 -aha -m Qwen/Qwen3-VL-2B-Instruct +aha cli -m Qwen/Qwen3-VL-2B-Instruct ``` ### 场景 2：使用已有模型启动服务 @@ -360,7 +368,7 @@ aha serv -m Qwen/Qwen3-VL-2B-Instruct --weight-path /data/models/Qwen/Qwen3-VL-2 ```bash # 在 0.0.0.0:8080 启动服务，允许外部访问 -aha -m Qwen/Qwen3-VL-2B-Instruct -a 0.0.0.0 -p 8080 +aha cli -m Qwen/Qwen3-VL-2B-Instruct -a 0.0.0.0 -p 8080 ``` ## API 接口 @@ -394,17 +402,6 @@ aha -m Qwen/Qwen3-VL-2B-Instruct -a 0.0.0.0 -p 8080 - **安全性**: 默认仅允许本地访问，使用 `--allow-remote-shutdown` 标志启用远程访问（不推荐） - **格式**: JSON 响应 -## 向后兼容性 - -为了保持与旧版本的兼容性，以下两种使用方式是等效的： - -```bash -# 新方式（推荐） -aha cli -m Qwen/Qwen3-VL-2B-Instruct - -# 旧方式（向后兼容） -aha -m Qwen/Qwen3-VL-2B-Instruct -``` ## 注意事项 diff --git a/docs/installation.md b/docs/installation.md index 55faff4..cb2e4a2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -291,7 +291,8 @@ OpenBMB/MiniCPM4-0.5B OpenBMB llm ✔ Qwen/Qwen2.5-VL-3B-Instruct Qwen vlm ✔ Qwen/Qwen2.5-VL-7B-Instruct Qwen vlm Qwen/Qwen3-0.6B Qwen llm ✔ -Qwen/Qwen3-1.7B Qwen llm ✔ +Qwen/Qwen3-1.7B Qwen llm +Qwen/Qwen3-4B Qwen llm Qwen/Qwen3.5-0.8B Qwen vlm ✔ Qwen/Qwen3.5-2B Qwen vlm Qwen/Qwen3.5-4B Qwen vlm diff --git a/docs/installation.zh-CN.md b/docs/installation.zh-CN.md index 7726e41..d28812b 100644 --- a/docs/installation.zh-CN.md +++ b/docs/installation.zh-CN.md @@ -291,7 +291,8 @@ OpenBMB/MiniCPM4-0.5B OpenBMB llm ✔ Qwen/Qwen2.5-VL-3B-Instruct Qwen vlm ✔ Qwen/Qwen2.5-VL-7B-Instruct Qwen vlm Qwen/Qwen3-0.6B Qwen llm ✔ -Qwen/Qwen3-1.7B Qwen llm ✔ +Qwen/Qwen3-1.7B Qwen llm +Qwen/Qwen3-4B Qwen llm Qwen/Qwen3.5-0.8B Qwen vlm ✔ Qwen/Qwen3.5-2B Qwen vlm Qwen/Qwen3.5-4B Qwen vlm diff --git a/docs/supported-models.md b/docs/supported-models.md index f7a1edb..7b585b9 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -15,7 +15,8 @@ OpenBMB/MiniCPM4-0.5B OpenBMB llm ✔ Qwen/Qwen2.5-VL-3B-Instruct Qwen vlm ✔ Qwen/Qwen2.5-VL-7B-Instruct Qwen vlm Qwen/Qwen3-0.6B Qwen llm ✔ -Qwen/Qwen3-1.7B Qwen llm ✔ +Qwen/Qwen3-1.7B Qwen llm +Qwen/Qwen3-4B Qwen llm Qwen/Qwen3.5-0.8B Qwen vlm ✔ Qwen/Qwen3.5-2B Qwen vlm Qwen/Qwen3.5-4B Qwen vlm @@ -44,7 +45,7 @@ ZhipuAI/GLM-OCR ZhipuAI ocr ✔ | Model | Parameters | Model Id | License | |-------|-----------|--------|---------| -| **Qwen3-0.6B** | 0.6B | Qwen/Qwen3-0.6B
Qwen/Qwen3-1.7B | [Apache 2.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md) | +| **Qwen3-0.6B** | 0.6B | Qwen/Qwen3-0.6B
Qwen/Qwen3-1.7B
Qwen/Qwen3-4B | [Apache 2.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md) | | **MiniCPM4-0.5B** | 0.5B | OpenBMB/MiniCPM4-0.5B | [Apache 2.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md) | | **LFM2-1.2B** | 1.2B | LiquidAI/LFM2-1.2B | [lfm1.0](https://huggingface.co/LiquidAI/LFM2-1.2B/blob/main/LICENSE) | | **LFM2.5-1.2B-Instruct** | 1.2B | LiquidAI/LFM2.5-1.2B-Instruct | [lfm1.0](https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct/blob/main/LICENSE) | diff --git a/docs/supported-models.zh-CN.md b/docs/supported-models.zh-CN.md index 071df14..0130eea 100644 --- a/docs/supported-models.zh-CN.md +++ b/docs/supported-models.zh-CN.md @@ -15,7 +15,8 @@ OpenBMB/MiniCPM4-0.5B OpenBMB llm ✔ Qwen/Qwen2.5-VL-3B-Instruct Qwen vlm ✔ Qwen/Qwen2.5-VL-7B-Instruct Qwen vlm Qwen/Qwen3-0.6B Qwen llm ✔ -Qwen/Qwen3-1.7B Qwen llm ✔ +Qwen/Qwen3-1.7B Qwen llm +Qwen/Qwen3-4B Qwen llm Qwen/Qwen3.5-0.8B Qwen vlm ✔ Qwen/Qwen3.5-2B Qwen vlm Qwen/Qwen3.5-4B Qwen vlm @@ -44,7 +45,7 @@ ZhipuAI/GLM-OCR ZhipuAI ocr ✔ | 模型 | 参数量 | 模型id | 开源协议 | |------|--------|------|---------| -| **Qwen3-0.6B** | 0.6B | Qwen/Qwen3-0.6B
Qwen/Qwen3-1.7B | [Apache 2.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md) | +| **Qwen3-0.6B** | 0.6B | Qwen/Qwen3-0.6B
Qwen/Qwen3-1.7B
Qwen/Qwen3-4B | [Apache 2.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md) | | **MiniCPM4-0.5B** | 0.5B | OpenBMB/MiniCPM4-0.5B | [Apache 2.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md) | | **LFM2-1.2B** | 1.2B | LiquidAI/LFM2-1.2B | [lfm1.0](https://huggingface.co/LiquidAI/LFM2-1.2B/blob/main/LICENSE) | | **LFM2.5-1.2B-Instruct** | 1.2B | LiquidAI/LFM2.5-1.2B-Instruct | [lfm1.0](https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct/blob/main/LICENSE) | diff --git a/src/cli/args.rs b/src/cli/args.rs index 6f313dc..d47c5f7 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -5,40 +5,8 @@ use clap::{Args, Parser, Subcommand}; #[command(name = "aha")] #[command(version, about, long_about = None)] pub(crate) struct Cli { - /// Service listen address - #[arg(short, long, default_value = "127.0.0.1")] - pub address: Option, - - /// Service listen port - #[arg(short, long)] - pub port: Option, - - /// Model type (required for backward compatibility) - #[arg(short, long)] - pub model: Option, - - /// Local model weight path - #[arg(long)] - pub weight_path: Option, - - /// Model download save directory - #[arg(long)] - pub save_dir: Option, - - /// Download retry count - #[arg(long)] - pub download_retries: Option, - - /// Local GGUF model weight path (required for loading models with GGUF). - #[arg(long)] - pub gguf_path: Option, - - /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) - #[arg(long)] - pub mmproj_path: Option, - #[command(subcommand)] - pub command: Option, + pub command: Commands, } #[derive(Subcommand, Debug)] @@ -61,7 +29,7 @@ pub(crate) enum Commands { /// Common/shared arguments for server operations #[derive(Args, Debug)] -pub(crate) struct CommonArgs { +pub(crate) struct ServerCommonArgs { /// Service listen address #[arg(short, long, default_value = "127.0.0.1")] pub address: String, @@ -70,24 +38,42 @@ pub(crate) struct CommonArgs { #[arg(short, long, default_value_t = 10100)] pub port: u16, - /// Model type (required) - #[arg(short, long)] - pub model: WhichModel, - /// Allow remote shutdown requests (default: local only, use with caution) #[arg(long)] pub allow_remote_shutdown: bool, } -/// Arguments for the 'cli' subcommand (download + serve) #[derive(Args, Debug)] -pub(crate) struct CliArgs { - #[command(flatten)] - pub common: CommonArgs, - +pub(crate) struct PathCommonArgs { /// Local model weight path (skip download if provided) #[arg(long)] pub weight_path: Option, + /// Local GGUF model weight path (required for loading models with GGUF). + #[arg(long)] + pub gguf_path: Option, + + /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) + #[arg(long)] + pub mmproj_path: Option, + + /// Local path for onnx model weights (required for loading with onnx) + #[arg(long)] + pub onnx_path: Option, + + /// config path for onnx/gguf model need extra config file + #[arg(long)] + pub config_path: Option, +} + +/// Arguments for the 'cli' subcommand (download + serve) +#[derive(Args, Debug)] +pub(crate) struct CliArgs { + /// Model type (required) + #[arg(short, long)] + pub model: WhichModel, + + #[command(flatten)] + pub server_common: ServerCommonArgs, /// Model download save directory #[arg(long)] @@ -97,32 +83,22 @@ pub(crate) struct CliArgs { #[arg(long)] pub download_retries: Option, - /// Local GGUF model weight path (required for loading models with GGUF). - #[arg(long)] - pub gguf_path: Option, - - /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) - #[arg(long)] - pub mmproj_path: Option, + #[command(flatten)] + pub path_common: PathCommonArgs, } /// Arguments for the 'serv start' subcommand #[derive(Args, Debug)] pub(crate) struct ServArgs { + /// Model type (required) + #[arg(short, long)] + pub model: WhichModel, + #[command(flatten)] - pub common: CommonArgs, + pub server_common: ServerCommonArgs, - /// Local model weight path (defaults to ~/.aha/{model_id} if not specified) - #[arg(long)] - pub weight_path: Option, - - /// Local GGUF model weight path (required for loading models with GGUF). - #[arg(long)] - pub gguf_path: Option, - - /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) - #[arg(long)] - pub mmproj_path: Option, + #[command(flatten)] + pub path_common: PathCommonArgs, } /// Arguments for the 'serv list' subcommand @@ -164,17 +140,8 @@ pub(crate) struct RunArgs { #[arg(short, long)] pub output: Option, - /// Local model weight path (defaults to ~/.aha/{model_id} if not specified) - #[arg(long)] - pub weight_path: Option, - - /// Local GGUF model weight path (required for loading models with GGUF). - #[arg(long)] - pub gguf_path: Option, - - /// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF) - #[arg(long)] - pub mmproj_path: Option, + #[command(flatten)] + pub path_common: PathCommonArgs, } /// Arguments for the 'delete' subcommand (delete model from default location) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 12d910d..221186a 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -76,22 +76,27 @@ pub(crate) fn run_list(args: ListArgs) -> anyhow::Result<()> { /// Run the 'cli' subcommand: download model (if needed) and start service pub(crate) async fn run_cli(args: CliArgs) -> anyhow::Result<()> { let CliArgs { - common, - weight_path, + model, + server_common, save_dir, download_retries, - gguf_path, - mmproj_path, + path_common, } = args; - let model_id = common.model.as_string(); + let model_id = model.as_string(); - let (model_path, gguf, mmproj) = if model_id.contains("gguf") { - if gguf_path.is_none() { + let (model_path, gguf, mmproj) = if model.is_gguf() { + if path_common.gguf_path.is_none() { return Err(anyhow!("gguf model path is required")); } - ("GGUF".to_string(), gguf_path, mmproj_path) + ( + "GGUF".to_string(), + path_common.gguf_path, + path_common.mmproj_path, + ) + } else if model.is_onnx() { + return Err(anyhow!("onnx model not support now")); } else { - let model_path = match weight_path { + let model_path = match path_common.weight_path { Some(path) => path, None => { let save_dir = match save_dir { @@ -106,8 +111,13 @@ pub(crate) async fn run_cli(args: CliArgs) -> anyhow::Result<()> { (model_path, None, None) }; - init(common.model, model_path, gguf, mmproj)?; - start_http_server(common.address, common.port, common.allow_remote_shutdown).await?; + init(model, model_path, gguf, mmproj)?; + start_http_server( + server_common.address, + server_common.port, + server_common.allow_remote_shutdown, + ) + .await?; Ok(()) } @@ -115,27 +125,41 @@ pub(crate) async fn run_cli(args: CliArgs) -> anyhow::Result<()> { /// Run the 'serv' subcommand: start service only (no download) pub(crate) async fn run_serv(args: ServArgs) -> anyhow::Result<()> { let ServArgs { - common, - weight_path, - gguf_path, - mmproj_path, + model, + server_common, + path_common, } = args; - let model_id = common.model.as_string(); - let (model_path, gguf, mmproj) = if model_id.contains("gguf") { - if gguf_path.is_none() { + let (model_path, gguf, mmproj) = if model.is_gguf() { + if path_common.gguf_path.is_none() { return Err(anyhow!("gguf model path is required")); } - ("GGUF".to_string(), gguf_path, mmproj_path) + ( + "GGUF".to_string(), + path_common.gguf_path, + path_common.mmproj_path, + ) + } else if model.is_onnx() { + return Err(anyhow!("onnx model not support now")); } else { - let model_path = match weight_path { + let model_path = match path_common.weight_path { Some(path) => path, - None => get_default_weight_path(common.model), + None => get_default_weight_path(model), }; + if !std::path::Path::new(&model_path).exists() { + return Err(anyhow!( + "serv subcommand will not download model, use `weight-path` to pass the model path" + )); + } (model_path, None, None) }; - init(common.model, model_path, gguf, mmproj)?; - start_http_server(common.address, common.port, common.allow_remote_shutdown).await?; + init(model, model_path, gguf, mmproj)?; + start_http_server( + server_common.address, + server_common.port, + server_common.allow_remote_shutdown, + ) + .await?; Ok(()) } @@ -205,13 +229,11 @@ pub(crate) fn run_run(args: RunArgs) -> anyhow::Result<()> { model, input, output, - weight_path, - gguf_path, - mmproj_path, + path_common, } = args; // Use default weight path if not specified - let weight_path = match weight_path { + let weight_path = match path_common.weight_path { Some(path) => path, None => get_default_weight_path(model), }; @@ -243,6 +265,9 @@ pub(crate) fn run_run(args: RunArgs) -> anyhow::Result<()> { WhichModel::Qwen3_1_7B => { qwen3::Qwen3Exec::run(&input, output.as_deref(), &weight_path)?; } + WhichModel::Qwen3_4B => { + qwen3::Qwen3Exec::run(&input, output.as_deref(), &weight_path)?; + } WhichModel::Qwen3_5_0_8B => { qwen3_5::Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?; } @@ -256,7 +281,12 @@ pub(crate) fn run_run(args: RunArgs) -> anyhow::Result<()> { qwen3_5::Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?; } WhichModel::Qwen3_5Gguf => { - qwen3_5::Qwen3_5Exec::run_gguf(&input, output.as_deref(), gguf_path, mmproj_path)?; + qwen3_5::Qwen3_5Exec::run_gguf( + &input, + output.as_deref(), + path_common.gguf_path, + path_common.mmproj_path, + )?; } WhichModel::Qwen3ASR0_6B => { qwen3_asr::Qwen3ASRExec::run(&input, output.as_deref(), &weight_path)?; diff --git a/src/main.rs b/src/main.rs index f6ee4c3..b8649e2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use clap::Parser; use crate::cli::{ - args::{Cli, CliArgs, Commands, CommonArgs}, + args::{Cli, Commands}, run_cli, run_delete, run_download, run_list, run_ps, run_run, run_serv, }; @@ -11,32 +11,13 @@ mod server; #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); - match cli.command { - Some(Commands::Cli(args)) => run_cli(args).await, - Some(Commands::Serv(args)) => run_serv(args).await, - Some(Commands::Ps(args)) => run_ps(args), - Some(Commands::Delete(args)) => run_delete(args), - Some(Commands::Download(args)) => run_download(args).await, - Some(Commands::Run(args)) => run_run(args), - Some(Commands::List(args)) => run_list(args), - None => { - // Backward compatibility: when no subcommand is provided, use 'cli' behavior - let model = cli.model.expect("Model is required (use -m or --model)"); - let args = CliArgs { - common: CommonArgs { - address: cli.address.unwrap_or_else(|| "127.0.0.1".to_string()), - port: cli.port.unwrap_or(10100), - model, - allow_remote_shutdown: false, - }, - weight_path: cli.weight_path, - save_dir: cli.save_dir, - download_retries: cli.download_retries, - gguf_path: cli.gguf_path, - mmproj_path: cli.mmproj_path, - }; - run_cli(args).await - } + Commands::Cli(args) => run_cli(args).await, + Commands::Serv(args) => run_serv(args).await, + Commands::Ps(args) => run_ps(args), + Commands::Delete(args) => run_delete(args), + Commands::Download(args) => run_download(args).await, + Commands::Run(args) => run_run(args), + Commands::List(args) => run_list(args), } } diff --git a/src/models/common/model_mapping.rs b/src/models/common/model_mapping.rs index 0fa5d99..291ac1f 100644 --- a/src/models/common/model_mapping.rs +++ b/src/models/common/model_mapping.rs @@ -20,6 +20,8 @@ pub enum WhichModel { Qwen3_0_6B, #[value(name = "Qwen/Qwen3-1.7B")] Qwen3_1_7B, + #[value(name = "Qwen/Qwen3-4B")] + Qwen3_4B, #[value(name = "Qwen/Qwen3.5-0.8B")] Qwen3_5_0_8B, #[value(name = "Qwen/Qwen3.5-2B")] @@ -67,18 +69,40 @@ pub enum WhichModel { } impl WhichModel { - /// Get the ModelScope model ID for this model variant + /// Get the model ID for this model variant pub fn as_string(&self) -> String { self.to_possible_value() .expect("not exists") .get_name() .to_string() } + + /// Checks if the model is in GGUF format + /// + /// Returns true if the model ID contains "gguf", false otherwise + pub fn is_gguf(&self) -> bool { + let model_id = self.as_string(); + model_id.to_lowercase().contains("gguf") + } + + /// Checks if the model is in ONNX format + /// + /// Returns true if the model ID contains "onnx", false otherwise + pub fn is_onnx(&self) -> bool { + let model_id = self.as_string(); + model_id.to_lowercase().contains("onnx") + } + /// Get the WhichModel enum list pub fn model_list() -> Vec { WhichModel::value_variants().to_vec() } + /// Extracts the model owner/organization from the model ID + /// + /// Splits the model ID string on '/' and returns the first part which typically represents + /// the organization or user who owns the model in Hugging Face format (e.g., "Qwen" from "Qwen/Qwen3-0.6B") + /// Returns "none" if the model ID doesn't contain a '/' separator pub fn model_owner(&self) -> String { let name = self.as_string(); let names: Vec<&str> = name.split("/").collect(); @@ -96,6 +120,7 @@ impl WhichModel { WhichModel::MiniCPM4_0_5B | WhichModel::Qwen3_0_6B | WhichModel::Qwen3_1_7B + | WhichModel::Qwen3_4B | WhichModel::LFM2_1_2B | WhichModel::LFM2_5_1_2BInstruct => "llm", // VLM models diff --git a/src/models/mod.rs b/src/models/mod.rs index b52fc86..aad8123 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -129,12 +129,27 @@ impl<'a> GenerateModel for ModelInstance<'a> { } } -pub fn load_model<'a>( +#[allow(unused)] +pub fn load_gguf_model<'a>( model_type: WhichModel, - path: &str, - gguf: Option<&str>, - mmproj: Option<&str>, + config_path: Option<&str>, // 有些gguf未包含模型其他配置，需额外指定 + gguf_path: &str, + mmproj_path: Option<&str>, ) -> Result> { + let model = match model_type { + WhichModel::Qwen3_5Gguf => { + let model = Qwen3_5GenerateModel::init_from_gguf(gguf_path, mmproj_path, None)?; + ModelInstance::Qwen3_5(model) + } + _ => { + let model_id = model_type.as_string(); + return Err(anyhow!("model id {model_id} is not gguf model")); + } + }; + Ok(model) +} + +pub fn load_model<'a>(model_type: WhichModel, path: &str) -> Result> { let model = match model_type { WhichModel::MiniCPM4_0_5B => { let model = MiniCPMGenerateModel::init(path, None, None)?; @@ -172,6 +187,10 @@ pub fn load_model<'a>( let model = Qwen3GenerateModel::init(path, None, None)?; ModelInstance::Qwen3(model) } + WhichModel::Qwen3_4B => { + let model = Qwen3GenerateModel::init(path, None, None)?; + ModelInstance::Qwen3(model) + } WhichModel::Qwen3_5_0_8B => { let model = Qwen3_5GenerateModel::init(path, None, None)?; ModelInstance::Qwen3_5(model) @@ -188,14 +207,6 @@ pub fn load_model<'a>( let model = Qwen3_5GenerateModel::init(path, None, None)?; ModelInstance::Qwen3_5(model) } - WhichModel::Qwen3_5Gguf => { - if gguf.is_none() { - return Err(anyhow!("Qwen3_5Gguf gguf model path is required")); - } - let gguf = gguf.unwrap(); - let model = Qwen3_5GenerateModel::init_from_gguf(gguf, mmproj, None)?; - ModelInstance::Qwen3_5(model) - } WhichModel::Qwen3ASR0_6B => { let model = Qwen3AsrGenerateModel::init(path, None, None)?; ModelInstance::Qwen3ASR(model) @@ -264,6 +275,10 @@ pub fn load_model<'a>( let model = GlmOcrGenerateModel::init(path, None, None)?; ModelInstance::GlmOCR(model) } + _ => { + let model_id = model_type.as_string(); + return Err(anyhow!("model id {model_id} is not safetensor model")); + } }; Ok(model) } diff --git a/src/models/qwen3_5/generate.rs b/src/models/qwen3_5/generate.rs index 7064719..04b02b5 100644 --- a/src/models/qwen3_5/generate.rs +++ b/src/models/qwen3_5/generate.rs @@ -59,7 +59,7 @@ impl<'a> Qwen3_5GenerateModel<'a> { qwen3_5, device, model_name: model_name.to_string(), - repeat_penalty: 1.01, + repeat_penalty: 1.0, repeat_last_n: 64, }) } @@ -116,7 +116,7 @@ impl<'a> Qwen3_5GenerateModel<'a> { qwen3_5, device, model_name: stem.to_string(), - repeat_penalty: 1.1, + repeat_penalty: 1.2, repeat_last_n: 64, }) } diff --git a/src/server/api.rs b/src/server/api.rs index a91e4a1..7dee106 100644 --- a/src/server/api.rs +++ b/src/server/api.rs @@ -2,9 +2,11 @@ use std::pin::pin; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; +use aha::models::load_gguf_model; use aha::models::{GenerateModel, ModelInstance, common::model_mapping::WhichModel, load_model}; use aha::params::chat::ChatCompletionParameters; use aha::utils::string_to_static_str; +use anyhow::anyhow; use rocket::futures::StreamExt; use rocket::serde::{Serialize, json::Json}; use rocket::{ @@ -37,10 +39,21 @@ pub fn init( gguf: Option, mmproj: Option, ) -> anyhow::Result<()> { - let model_path = string_to_static_str(path); - let gguf = gguf.map(string_to_static_str); - let mmproj = mmproj.map(string_to_static_str); - let model = load_model(model_type, model_path, gguf, mmproj)?; + let model = if model_type.is_gguf() { + if let Some(gguf_path) = gguf { + let gguf_path = string_to_static_str(gguf_path); + let mmproj_path = mmproj.map(string_to_static_str); + load_gguf_model(model_type, None, gguf_path, mmproj_path)? + } else { + return Err(anyhow!("gguf model need gguf model path")); + } + } else if model_type.is_onnx() { + return Err(anyhow!("onnx comming soon but now not support")); + } else { + let model_path = string_to_static_str(path); + load_model(model_type, model_path)? + }; + MODEL.get_or_init(|| { Arc::new(RwLock::new(StoredModel { which_model: model_type,