```
feat(cli): add direct model inference via new run subcommand - Add `aha run` CLI subcommand for direct model inference without HTTP service - Support multiple models including Qwen series, OCR models, ASR models, and voice generation - Implement input/output handling with file path support and auto-generation - Add comprehensive documentation in CLI_USAGE.md with examples - Include performance timing for model loading and inference operations - Add macOS build target to Makefile with Metal support ```
This commit is contained in:
+7
-8
@@ -1,18 +1,17 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **CLI Subcommand Support**: Added three new subcommands for better command organization:
|
||||
- `aha cli` - Download model and start HTTP service (default, backward compatible)
|
||||
- `aha serv` - Start HTTP service only (requires `--weight-path`)
|
||||
- `aha download` - Download model only (no service start)
|
||||
- **CLI `run` Subcommand**: Direct model inference from CLI without HTTP service overhead:
|
||||
- `aha run` - Run model inference directly
|
||||
- `-m, --model <MODEL>` - Specify which model to use
|
||||
- `-in, --input <INPUT>` - Input text or file path (model-specific interpretation)
|
||||
- `-out, --output <OUTPUT>` - Output file path (optional, auto-generated if not specified)
|
||||
- `--weight-path <WEIGHT_PATH>` - Local model weight path (required)
|
||||
|
||||
|
||||
### Changed
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ build:
|
||||
@echo "Building project..."
|
||||
@cargo build
|
||||
|
||||
build_mac:
|
||||
@echo "Building project for macOS..."
|
||||
@cargo build --features metal --release
|
||||
|
||||
test:
|
||||
@echo "Running tests..."
|
||||
@cargo test
|
||||
|
||||
@@ -59,6 +59,46 @@ aha cli -m qwen3vl-2b --weight-path /path/to/model
|
||||
aha -m qwen3vl-2b
|
||||
```
|
||||
|
||||
### run - 直接模型推理
|
||||
|
||||
直接运行模型推理,无需启动 HTTP 服务。适用于一次性推理任务或批处理。
|
||||
|
||||
**语法:**
|
||||
```bash
|
||||
aha run [OPTIONS] --model <MODEL> --input <INPUT> --weight-path <WEIGHT_PATH>
|
||||
```
|
||||
|
||||
**选项:**
|
||||
|
||||
| 选项 | 说明 | 默认值 |
|
||||
|------|------|--------|
|
||||
| `-m, --model <MODEL>` | 模型类型(必选) | - |
|
||||
| `-in, --input <INPUT>` | 输入文本或文件路径(模型特定解释) | - |
|
||||
| `-out, --output <OUTPUT>` | 输出文件路径(可选,未指定则自动生成) | - |
|
||||
| `--weight-path <WEIGHT_PATH>` | 本地模型权重路径(必选) | - |
|
||||
|
||||
**示例:**
|
||||
|
||||
```bash
|
||||
# VoxCPM1.5 文字转语音
|
||||
aha run -m voxcpm1.5 -in "太阳当空照" -out output.wav --weight-path /path/to/model
|
||||
|
||||
# VoxCPM1.5 从文件读取输入
|
||||
aha run -m voxcpm1.5 -in "file://./input.txt" --weight-path /path/to/model
|
||||
|
||||
# MiniCPM4 文本生成
|
||||
aha run -m minicpm4-0.5b -in "你好" --weight-path /path/to/model
|
||||
|
||||
# DeepSeek OCR 图片识别
|
||||
aha run -m deepseek-ocr -in "image.jpg" --weight-path /path/to/model
|
||||
|
||||
# RMBG2.0 背景移除
|
||||
aha run -m RMBG2.0 -in "photo.png" -out "no_bg.png" --weight-path /path/to/model
|
||||
|
||||
# GLM-ASR 语音识别
|
||||
aha run -m glm-asr-nano-2512 -in "audio.wav" -in "请转写这段音频" --weight-path /path/to/model
|
||||
```
|
||||
|
||||
### serv - 启动服务
|
||||
|
||||
仅启动 HTTP 服务,不下载模型。必须通过 `--weight-path` 指定本地模型路径。
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
//! DeepSeek-OCR exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, deepseek_ocr::generate::DeepseekOCRGenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct DeepSeekORExec;
|
||||
|
||||
impl ExecModel for DeepSeekORExec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let input_path = if input.starts_with("file://") {
|
||||
input.to_string()
|
||||
} else {
|
||||
format!("file://{}", input)
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = DeepseekOCRGenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "deepseek-ocr",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{{
|
||||
"type": "image",
|
||||
"image_url": {{
|
||||
"url": "{}"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
input_path
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let result = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
println!("Result: {:?}", result);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", result))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
//! Fun-ASR-Nano-2512 exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, fun_asr_nano::generate::FunAsrNanoGenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct FunASRNanoExec;
|
||||
|
||||
impl ExecModel for FunASRNanoExec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let target_text = if input.starts_with("file://") {
|
||||
let path = &input[7..];
|
||||
std::fs::read_to_string(path)?
|
||||
} else {
|
||||
input.to_string()
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = FunAsrNanoGenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
// Create ChatCompletionParameters for ASR
|
||||
let input_url = if input.starts_with("http://") || input.starts_with("https://") || input.starts_with("file://") {
|
||||
input.to_string()
|
||||
} else {
|
||||
format!("file://{}", input)
|
||||
};
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "fun-asr-nano",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{{
|
||||
"type": "audio",
|
||||
"audio_url": {{
|
||||
"url": "{}"
|
||||
}}
|
||||
}},
|
||||
{{
|
||||
"type": "text",
|
||||
"text": "{}"
|
||||
}}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
input_url, target_text
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let res = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
println!("Result: {:?}", res);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", res))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
//! GLM-ASR-Nano-2512 exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, glm_asr_nano::generate::GlmAsrNanoGenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct GlmASRNanoExec;
|
||||
|
||||
impl ExecModel for GlmASRNanoExec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let target_text = if input.starts_with("file://") {
|
||||
let path = &input[7..];
|
||||
std::fs::read_to_string(path)?
|
||||
} else {
|
||||
input.to_string()
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = GlmAsrNanoGenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
// Create ChatCompletionParameters for ASR
|
||||
// Input should be an audio file path
|
||||
let input_url = if input.starts_with("http://") || input.starts_with("https://") || input.starts_with("file://") {
|
||||
input.to_string()
|
||||
} else {
|
||||
format!("file://{}", input)
|
||||
};
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "glm-asr-nano",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{{
|
||||
"type": "audio",
|
||||
"audio_url": {{
|
||||
"url": "{}"
|
||||
}}
|
||||
}},
|
||||
{{
|
||||
"type": "text",
|
||||
"text": "{}"
|
||||
}}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
input_url, target_text
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let res = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
println!("Result: {:?}", res);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", res))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
//! Hunyuan-OCR exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, hunyuan_ocr::generate::HunyuanOCRGenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct HunyuanORExec;
|
||||
|
||||
impl ExecModel for HunyuanORExec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let input_path = if input.starts_with("file://") {
|
||||
input.to_string()
|
||||
} else {
|
||||
format!("file://{}", input)
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = HunyuanOCRGenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "hunyuan-ocr",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{{
|
||||
"type": "image",
|
||||
"image_url": {{
|
||||
"url": "{}"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
input_path
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let result = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
println!("Result: {:?}", result);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", result))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
//! MiniCPM4-0.5B exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, minicpm4::generate::MiniCPMGenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct MiniCPM4Exec;
|
||||
|
||||
impl ExecModel for MiniCPM4Exec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let target_text = if input.starts_with("file://") {
|
||||
let path = &input[7..];
|
||||
std::fs::read_to_string(path)?
|
||||
} else {
|
||||
input.to_string()
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = MiniCPMGenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"temperature": 0.3,
|
||||
"top_p": 0.8,
|
||||
"model": "minicpm4",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": "{}"
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
target_text.replace('"', "\\\"")
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let result = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
// Print result
|
||||
println!("Result: {:?}", result);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", result))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
//! CLI exec module for direct model inference
|
||||
//!
|
||||
//! This module provides model-specific exec implementations for the `run` subcommand.
|
||||
//! Each model has its own exec module that handles input/output parsing and model invocation.
|
||||
|
||||
pub mod deepseek_ocr;
|
||||
pub mod fun_asr_nano;
|
||||
pub mod glm_asr_nano;
|
||||
pub mod hunyuan_ocr;
|
||||
pub mod minicpm4;
|
||||
pub mod paddleocr_vl;
|
||||
pub mod qwen2_5vl;
|
||||
pub mod qwen3;
|
||||
pub mod qwen3vl;
|
||||
pub mod rmbg2_0;
|
||||
pub mod voxcpm;
|
||||
pub mod voxcpm1_5;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
/// Trait for model exec implementations
|
||||
///
|
||||
/// Each model exec module implements this trait to provide
|
||||
/// model-specific inference logic for CLI `run` commands.
|
||||
pub trait ExecModel {
|
||||
/// Run inference with the given input and output parameters
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `input` - Input text or file path (interpretation is model-specific)
|
||||
/// * `output` - Optional output file path (if None, model will auto-generate)
|
||||
/// * `weight_path` - Path to the model weights
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(())` on success
|
||||
/// * `Err(anyhow::Error)` on failure
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()>;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
//! PaddleOCR-VL exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, paddleocr_vl::generate::PaddleOCRVLGenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct PaddleOVLExec;
|
||||
|
||||
impl ExecModel for PaddleOVLExec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let input_path = if input.starts_with("file://") {
|
||||
input.to_string()
|
||||
} else {
|
||||
format!("file://{}", input)
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = PaddleOCRVLGenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "paddleocr-vl",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{{
|
||||
"type": "image",
|
||||
"image_url": {{
|
||||
"url": "{}"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
input_path
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let result = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
println!("Result: {:?}", result);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", result))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//! Qwen2.5VL-3B exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, qwen2_5vl::generate::Qwen2_5VLGenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct Qwen2_5vlExec;
|
||||
|
||||
impl ExecModel for Qwen2_5vlExec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let target_text = if input.starts_with("file://") {
|
||||
let path = &input[7..];
|
||||
std::fs::read_to_string(path)?
|
||||
} else {
|
||||
input.to_string()
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = Qwen2_5VLGenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "qwen2.5vl",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": "{}"
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
target_text.replace('"', "\\\"")
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let result = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
println!("Result: {:?}", result);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", result))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//! Qwen3-0.6B exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, qwen3::generate::Qwen3GenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct Qwen3Exec;
|
||||
|
||||
impl ExecModel for Qwen3Exec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let target_text = if input.starts_with("file://") {
|
||||
let path = &input[7..];
|
||||
std::fs::read_to_string(path)?
|
||||
} else {
|
||||
input.to_string()
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = Qwen3GenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "qwen3",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": "{}"
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
target_text.replace('"', "\\\"")
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let result = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
println!("Result: {:?}", result);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", result))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//! Qwen3VL-2B exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::{GenerateModel, qwen3vl::generate::Qwen3VLGenerateModel};
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct Qwen3vlExec;
|
||||
|
||||
impl ExecModel for Qwen3vlExec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let target_text = if input.starts_with("file://") {
|
||||
let path = &input[7..];
|
||||
std::fs::read_to_string(path)?
|
||||
} else {
|
||||
input.to_string()
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut model = Qwen3VLGenerateModel::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "qwen3vl",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": "{}"
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
target_text.replace('"', "\\\"")
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let result = model.generate(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
println!("Result: {:?}", result);
|
||||
|
||||
if let Some(out) = output {
|
||||
std::fs::write(out, format!("{:?}", result))?;
|
||||
println!("Output saved to: {}", out);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
//! RMBG2.0 exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::rmbg2_0::generate::RMBG2_0Model;
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct RMBG2_0Exec;
|
||||
|
||||
impl ExecModel for RMBG2_0Exec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let input_path = if input.starts_with("file://") {
|
||||
input.to_string()
|
||||
} else {
|
||||
format!("file://{}", input)
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let model = RMBG2_0Model::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
// Create ChatCompletionParameters for image background removal
|
||||
let message = format!(
|
||||
r#"{{
|
||||
"model": "rmbg2.0",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{{
|
||||
"type": "image",
|
||||
"image_url": {{
|
||||
"url": "{}"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}}"#,
|
||||
input_path
|
||||
);
|
||||
let mes = serde_json::from_str(&message)?;
|
||||
|
||||
let i_start = Instant::now();
|
||||
let result = model.inference(mes)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
let output_path = if let Some(out) = output {
|
||||
out.to_string()
|
||||
} else {
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)?
|
||||
.as_secs();
|
||||
format!("rmbg_{}.png", timestamp)
|
||||
};
|
||||
|
||||
// Save all result images
|
||||
for (i, img) in result.iter().enumerate() {
|
||||
let path = if result.len() == 1 {
|
||||
output_path.clone()
|
||||
} else {
|
||||
format!("{}_{}.png", output_path.trim_end_matches(".png"), i)
|
||||
};
|
||||
img.save(&path)?;
|
||||
println!("Output saved to: {}", path);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
//! VoxCPM exec implementation for CLI `run` subcommand
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::voxcpm::generate::VoxCPMGenerate;
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct VoxCPMExec;
|
||||
|
||||
impl ExecModel for VoxCPMExec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let target_text = if input.starts_with("file://") {
|
||||
let path = &input[7..];
|
||||
std::fs::read_to_string(path)?
|
||||
} else {
|
||||
input.to_string()
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut voxcpm_generate = VoxCPMGenerate::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let i_start = Instant::now();
|
||||
let audio = voxcpm_generate.inference(
|
||||
target_text,
|
||||
Some("啥子小师叔,打狗还要看主人,你再要继续,我就是你的对手".to_string()), //todo args
|
||||
Some("file://./assets/audio/voice_01.wav".to_string()), //todo args
|
||||
2,
|
||||
100, // max_len (voxcpm uses 100 vs voxcpm1.5's 4096)
|
||||
10,
|
||||
2.0,
|
||||
6.0,
|
||||
)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
let output_path = if let Some(out) = output {
|
||||
out.to_string()
|
||||
} else {
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)?
|
||||
.as_secs();
|
||||
format!("voxcpm_{}.wav", timestamp)
|
||||
};
|
||||
|
||||
let sample_rate = voxcpm_generate.sample_rate();
|
||||
crate::utils::audio_utils::save_wav(&audio, &output_path, sample_rate as u32)?;
|
||||
|
||||
println!("Output saved to: {}", output_path);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//! VoxCPM1.5 exec implementation for CLI `run` subcommand
|
||||
//!
|
||||
//! This module handles VoxCPM1.5 model inference for direct CLI execution.
|
||||
//! Input/output parameter interpretation is handled here as per the design:
|
||||
//! - Input can be text content or a file path (with `file://` prefix)
|
||||
//! - Output can be a file path or will be auto-generated if not specified
|
||||
|
||||
use crate::exec::ExecModel;
|
||||
use crate::models::voxcpm::generate::VoxCPMGenerate;
|
||||
use anyhow::{Ok, Result};
|
||||
use std::time::Instant;
|
||||
|
||||
pub struct VoxCPM1_5Exec;
|
||||
|
||||
impl ExecModel for VoxCPM1_5Exec {
|
||||
fn run(input: &str, output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||
let target_text = if input.starts_with("file://") {
|
||||
let path = &input[7..];
|
||||
std::fs::read_to_string(path)?
|
||||
} else {
|
||||
input.to_string()
|
||||
};
|
||||
|
||||
let i_start = Instant::now();
|
||||
let mut voxcpm_generate = VoxCPMGenerate::init(weight_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let i_start = Instant::now();
|
||||
let audio = voxcpm_generate.inference(
|
||||
target_text,
|
||||
Some("啥子小师叔,打狗还要看主人,你再要继续,我就是你的对手".to_string()), //todo args
|
||||
Some("file://./assets/audio/voice_01.wav".to_string()), //todo args
|
||||
2,
|
||||
4096,
|
||||
10,
|
||||
2.0,
|
||||
6.0,
|
||||
)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
|
||||
let output_path = if let Some(out) = output {
|
||||
out.to_string()
|
||||
} else {
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)?
|
||||
.as_secs();
|
||||
format!("voxcpm1_5_{}.wav", timestamp)
|
||||
};
|
||||
|
||||
let sample_rate = voxcpm_generate.sample_rate();
|
||||
crate::utils::audio_utils::save_wav(&audio, &output_path, sample_rate as u32)?;
|
||||
|
||||
println!("Output saved to: {}", output_path);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod chat_template;
|
||||
pub mod exec;
|
||||
pub mod models;
|
||||
pub mod position_embed;
|
||||
pub mod tokenizer;
|
||||
|
||||
+99
@@ -53,6 +53,8 @@ enum Commands {
|
||||
Serv(ServArgs),
|
||||
/// Download model only
|
||||
Download(DownloadArgs),
|
||||
/// Run model inference directly
|
||||
Run(RunArgs),
|
||||
}
|
||||
|
||||
/// Common/shared arguments for server operations
|
||||
@@ -117,6 +119,26 @@ struct DownloadArgs {
|
||||
download_retries: Option<u32>,
|
||||
}
|
||||
|
||||
/// Arguments for the 'run' subcommand (direct inference)
|
||||
#[derive(Args, Debug)]
|
||||
struct RunArgs {
|
||||
/// Model type (required)
|
||||
#[arg(short, long)]
|
||||
model: WhichModel,
|
||||
|
||||
/// Input text or file path
|
||||
#[arg(short, long)]
|
||||
input: String,
|
||||
|
||||
/// Output file path (optional)
|
||||
#[arg(short, long)]
|
||||
output: Option<String>,
|
||||
|
||||
/// Local model weight path (required)
|
||||
#[arg(long, required = true)]
|
||||
weight_path: String,
|
||||
}
|
||||
|
||||
async fn download_model(model_id: &str, save_dir: &str, max_retries: u32) -> anyhow::Result<()> {
|
||||
let mut attempts = 0u32;
|
||||
loop {
|
||||
@@ -222,6 +244,82 @@ async fn run_download(args: DownloadArgs) -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the 'run' subcommand: direct model inference
|
||||
fn run_run(args: RunArgs) -> anyhow::Result<()> {
|
||||
use aha::exec::ExecModel;
|
||||
|
||||
let RunArgs { model, input, output, weight_path } = args;
|
||||
|
||||
match model {
|
||||
WhichModel::MiniCPM4_0_5B => {
|
||||
use aha::exec::minicpm4::MiniCPM4Exec;
|
||||
MiniCPM4Exec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::Qwen2_5vl3B => {
|
||||
use aha::exec::qwen2_5vl::Qwen2_5vlExec;
|
||||
Qwen2_5vlExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::Qwen2_5vl7B => {
|
||||
use aha::exec::qwen2_5vl::Qwen2_5vlExec;
|
||||
Qwen2_5vlExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::Qwen3_0_6B => {
|
||||
use aha::exec::qwen3::Qwen3Exec;
|
||||
Qwen3Exec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::Qwen3vl2B => {
|
||||
use aha::exec::qwen3vl::Qwen3vlExec;
|
||||
Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::Qwen3vl4B => {
|
||||
use aha::exec::qwen3vl::Qwen3vlExec;
|
||||
Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::Qwen3vl8B => {
|
||||
use aha::exec::qwen3vl::Qwen3vlExec;
|
||||
Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::Qwen3vl32B => {
|
||||
use aha::exec::qwen3vl::Qwen3vlExec;
|
||||
Qwen3vlExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::DeepSeekOCR => {
|
||||
use aha::exec::deepseek_ocr::DeepSeekORExec;
|
||||
DeepSeekORExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::HunyuanOCR => {
|
||||
use aha::exec::hunyuan_ocr::HunyuanORExec;
|
||||
HunyuanORExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::PaddleOCRVL => {
|
||||
use aha::exec::paddleocr_vl::PaddleOVLExec;
|
||||
PaddleOVLExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::RMBG2_0 => {
|
||||
use aha::exec::rmbg2_0::RMBG2_0Exec;
|
||||
RMBG2_0Exec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::VoxCPM => {
|
||||
use aha::exec::voxcpm::VoxCPMExec;
|
||||
VoxCPMExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::VoxCPM1_5 => {
|
||||
use aha::exec::voxcpm1_5::VoxCPM1_5Exec;
|
||||
VoxCPM1_5Exec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::GlmASRNano2512 => {
|
||||
use aha::exec::glm_asr_nano::GlmASRNanoExec;
|
||||
GlmASRNanoExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
WhichModel::FunASRNano2512 => {
|
||||
use aha::exec::fun_asr_nano::FunASRNanoExec;
|
||||
FunASRNanoExec::run(&input, output.as_deref(), &weight_path)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
@@ -230,6 +328,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
Some(Commands::Cli(args)) => run_cli(args).await,
|
||||
Some(Commands::Serv(args)) => run_serv(args).await,
|
||||
Some(Commands::Download(args)) => run_download(args).await,
|
||||
Some(Commands::Run(args)) => run_run(args),
|
||||
None => {
|
||||
// Backward compatibility: when no subcommand is provided, use 'cli' behavior
|
||||
let model = cli.model.expect("Model is required (use -m or --model)");
|
||||
|
||||
Reference in New Issue
Block a user