tckm.yan/aha
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cli add about gguf param

Browse Source
This commit is contained in:
jhqxxx
2026-03-17 15:39:34 +08:00
parent 80d36b5308
commit fb4a50f745
19 changed files with 403 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/mod.rs
+12 -3
View File
@@ -37,9 +37,16 @@ static SHUTDOWN_FLAG: OnceLock<Arc<AtomicBool>> = OnceLock::new();
static SERVER_PORT: OnceLock<u16> = OnceLock::new();
static ALLOW_REMOTE_SHUTDOWN: OnceLock<bool> = OnceLock::new();
pub fn init(model_type: WhichModel, path: String) -> anyhow::Result<()> {
pub fn init(
model_type: WhichModel,
path: String,
gguf: Option<String>,
mmproj: Option<String>,
) -> anyhow::Result<()> {
let model_path = string_to_static_str(path);
let model = load_model(model_type, model_path)?;
let gguf = gguf.map(string_to_static_str);
let mmproj = mmproj.map(string_to_static_str);
let model = load_model(model_type, model_path, gguf, mmproj)?;
MODEL.get_or_init(|| {
Arc::new(RwLock::new(StoredModel {
which_model: model_type,
@@ -247,6 +254,7 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str {
WhichModel::Qwen3_5_2B => "qwen3.5-2b",
WhichModel::Qwen3_5_4B => "qwen3.5-4b",
WhichModel::Qwen3_5_9B => "qwen3.5-9b",
WhichModel::Qwen3_5Gguf => "qwen3.5-gguf",
WhichModel::Qwen3ASR0_6B => "qwen3asr-0.6b",
WhichModel::Qwen3ASR1_7B => "qwen3asr-1.7b",
WhichModel::Qwen3vl2B => "qwen3vl-2b",
@@ -274,7 +282,8 @@ fn which_model_to_owner(which_model: WhichModel) -> &'static str {
WhichModel::Qwen3vl2B
| WhichModel::Qwen3vl4B
| WhichModel::Qwen3vl8B
| WhichModel::Qwen3vl32B => "Qwen",
| WhichModel::Qwen3vl32B
| WhichModel::Qwen3_5Gguf => "Qwen",
WhichModel::Qwen3_5_0_8B
| WhichModel::Qwen3_5_2B
| WhichModel::Qwen3_5_4B
src/exec/qwen3_5.rs
+126 -3
View File
@@ -2,15 +2,138 @@
use std::time::Instant;
use anyhow::Result;
use anyhow::{Result, anyhow};
use crate::exec::ExecModel;
use crate::models::GenerateModel;
use crate::models::qwen3_5::generate::Qwen3_5GenerateModel;
use crate::utils::get_file_path;
use crate::utils::{get_file_path, string_to_static_str};
pub struct Qwen3_5Exec;
impl Qwen3_5Exec {
pub fn run_gguf(
input: &[String],
output: Option<&str>,
gguf_path: Option<String>,
mmproj_path: Option<String>,
) -> Result<()> {
let input_text = &input[0];
let target_text = if input_text.starts_with("file://") {
let path = get_file_path(input_text)?;
std::fs::read_to_string(path)?
} else {
input_text.clone()
};
let model_file = if let Some(g) = gguf_path {
g
} else {
return Err(anyhow!("gguf model path is required"));
};
let mmproj_path = mmproj_path.map(string_to_static_str);
let i_start = Instant::now();
let mut model = Qwen3_5GenerateModel::init_from_gguf(&model_file, mmproj_path, None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
let url = input.get(1);
let input_url = if let Some(url) = url
&& (url.starts_with("http://")
|| url.starts_with("https://")
|| url.starts_with("file://"))
{
Some(url.clone())
} else {
url.map(|url| format!("file://{}", url))
};
let message = if let Some(input_url) = &input_url
&& input_url.ends_with("mp4")
{
format!(
r#"{{
"model": "qwen3.5",
"messages": [
{{
"role": "user",
"content": [
{{
"type": "video",
"video_url":
{{
"url": "{}"
}}
}},
{{
"type": "text",
"text": "{}"
}}
]
}}
]
}}"#,
input_url, target_text
)
} else if let Some(input_url) = &input_url {
format!(
r#"{{
"model": "qwen3.5",
"messages": [
{{
"role": "user",
"content": [
{{
"type": "image",
"image_url": {{
"url": "{}"
}}
}},
{{
"type": "text",
"text": "{}"
}}
]
}}
]
}}"#,
input_url, target_text
)
} else {
format!(
r#"{{
"model": "qwen3.5",
"messages": [
{{
"role": "user",
"content": [
{{
"type": "text",
"text": "{}"
}}
]
}}
]
}}"#,
target_text
)
};
let mes = serde_json::from_str(&message)?;
let i_start = Instant::now();
let result = model.generate(mes)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in generate is: {:?}", i_duration);
println!("Result: {:?}", result);
if let Some(out) = output {
std::fs::write(out, format!("{:?}", result))?;
println!("Output saved to: {}", out);
}
Ok(())
}
}
impl ExecModel for Qwen3_5Exec {
fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
let input_text = &input[0];
@@ -62,7 +185,7 @@ impl ExecModel for Qwen3_5Exec {
} else {
format!(
r#"{{
"model": "qwen2.5",
"model": "qwen3.5",
"messages": [
{{
"role": "user",
src/exec/qwen3vl.rs
+29 -8
View File
@@ -24,16 +24,19 @@ impl ExecModel for Qwen3vlExec {
let mut model = Qwen3VLGenerateModel::init(weight_path, None, None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
let url = &input[1];
let input_url = if url.starts_with("http://")
|| url.starts_with("https://")
|| url.starts_with("file://")
let url = input.get(1);
let input_url = if let Some(url) = url
&& (url.starts_with("http://")
|| url.starts_with("https://")
|| url.starts_with("file://"))
{
url.clone()
Some(url.clone())
} else {
format!("file://{}", url)
url.map(|url| format!("file://{}", url))
};
let message = if input_url.ends_with("mp4") {
let message = if let Some(input_url) = &input_url
&& input_url.ends_with("mp4")
{
format!(
r#"{{
"model": "qwen3vl",
@@ -58,7 +61,7 @@ impl ExecModel for Qwen3vlExec {
}}"#,
input_url, target_text
)
} else {
} else if let Some(input_url) = &input_url {
format!(
r#"{{
"model": "qwen3vl",
@@ -82,6 +85,24 @@ impl ExecModel for Qwen3vlExec {
}}"#,
input_url, target_text
)
} else {
format!(
r#"{{
"model": "qwen3vl",
"messages": [
{{
"role": "user",
"content": [
{{
"type": "text",
"text": "{}"
}}
]
}}
]
}}"#,
target_text
)
};
let mes = serde_json::from_str(&message)?;
src/main.rs
+77 -17
View File
@@ -6,6 +6,7 @@ use aha::{
process::{cleanup_pid_file, create_pid_file},
utils::{download_model, get_default_save_dir},
};
use anyhow::anyhow;
use clap::{Args, Parser, Subcommand, ValueEnum};
use rocket::{
Config,
@@ -45,6 +46,14 @@ struct Cli {
#[arg(long)]
download_retries: Option<u32>,
/// Local GGUF model weight path (required for loading models with GGUF).
#[arg(long)]
gguf_path: Option<String>,
/// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF)
#[arg(long)]
mmproj_path: Option<String>,
#[command(subcommand)]
command: Option<Commands>,
}
@@ -104,6 +113,14 @@ struct CliArgs {
/// Download retry count
#[arg(long)]
download_retries: Option<u32>,
/// Local GGUF model weight path (required for loading models with GGUF).
#[arg(long)]
gguf_path: Option<String>,
/// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF)
#[arg(long)]
mmproj_path: Option<String>,
}
/// Arguments for the 'serv start' subcommand
@@ -115,6 +132,14 @@ struct ServArgs {
/// Local model weight path (defaults to ~/.aha/{model_id} if not specified)
#[arg(long)]
weight_path: Option<String>,
/// Local GGUF model weight path (required for loading models with GGUF).
#[arg(long)]
gguf_path: Option<String>,
/// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF)
#[arg(long)]
mmproj_path: Option<String>,
}
/// Arguments for the 'serv list' subcommand
@@ -159,6 +184,14 @@ struct RunArgs {
/// Local model weight path (defaults to ~/.aha/{model_id} if not specified)
#[arg(long)]
weight_path: Option<String>,
/// Local GGUF model weight path (required for loading models with GGUF).
#[arg(long)]
gguf_path: Option<String>,
/// Local path for mmproj GGUF model weights (required for loading with multimodel GGUF)
#[arg(long)]
mmproj_path: Option<String>,
}
/// Arguments for the 'delete' subcommand (delete model from default location)
@@ -282,23 +315,33 @@ async fn run_cli(args: CliArgs) -> anyhow::Result<()> {
weight_path,
save_dir,
download_retries,
gguf_path,
mmproj_path,
} = args;
let model_id = common.model.model_id();
let model_path = match weight_path {
Some(path) => path,
None => {
let save_dir = match save_dir {
Some(dir) => dir,
None => get_default_save_dir().expect("Failed to get home directory"),
};
let max_retries = download_retries.unwrap_or(3);
download_model(model_id, &save_dir, max_retries).await?;
save_dir + "/" + model_id
let (model_path, gguf, mmproj) = if model_id.eq("GGUF") {
if gguf_path.is_none() {
return Err(anyhow!("gguf model path is required"));
}
("GGUF".to_string(), gguf_path, mmproj_path)
} else {
let model_path = match weight_path {
Some(path) => path,
None => {
let save_dir = match save_dir {
Some(dir) => dir,
None => get_default_save_dir().expect("Failed to get home directory"),
};
let max_retries = download_retries.unwrap_or(3);
download_model(model_id, &save_dir, max_retries).await?;
save_dir + "/" + model_id
}
};
(model_path, None, None)
};
init(common.model, model_path)?;
init(common.model, model_path, gguf, mmproj)?;
start_http_server(common.address, common.port, common.allow_remote_shutdown).await?;
Ok(())
@@ -309,14 +352,24 @@ async fn run_serv(args: ServArgs) -> anyhow::Result<()> {
let ServArgs {
common,
weight_path,
gguf_path,
mmproj_path,
} = args;
let model_path = match weight_path {
Some(path) => path,
None => get_default_weight_path(common.model),
let model_id = common.model.model_id();
let (model_path, gguf, mmproj) = if model_id.eq("GGUF") {
if gguf_path.is_none() {
return Err(anyhow!("gguf model path is required"));
}
("GGUF".to_string(), gguf_path, mmproj_path)
} else {
let model_path = match weight_path {
Some(path) => path,
None => get_default_weight_path(common.model),
};
(model_path, None, None)
};
init(common.model, model_path)?;
init(common.model, model_path, gguf, mmproj)?;
start_http_server(common.address, common.port, common.allow_remote_shutdown).await?;
Ok(())
@@ -392,6 +445,8 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
input,
output,
weight_path,
gguf_path,
mmproj_path,
} = args;
// Use default weight path if not specified
@@ -399,7 +454,6 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
Some(path) => path,
None => get_default_weight_path(model),
};
match model {
WhichModel::MiniCPM4_0_5B => {
use aha::exec::minicpm4::MiniCPM4Exec;
@@ -433,6 +487,10 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
use aha::exec::qwen3_5::Qwen3_5Exec;
Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
}
WhichModel::Qwen3_5Gguf => {
use aha::exec::qwen3_5::Qwen3_5Exec;
Qwen3_5Exec::run_gguf(&input, output.as_deref(), gguf_path, mmproj_path)?;
}
WhichModel::Qwen3ASR0_6B => {
use aha::exec::qwen3_asr::Qwen3ASRExec;
Qwen3ASRExec::run(&input, output.as_deref(), &weight_path)?;
@@ -610,6 +668,8 @@ async fn main() -> anyhow::Result<()> {
weight_path: cli.weight_path,
save_dir: cli.save_dir,
download_retries: cli.download_retries,
gguf_path: cli.gguf_path,
mmproj_path: cli.mmproj_path,
};
run_cli(args).await
}
src/models/common/gguf.rs
+4
View File
@@ -213,6 +213,10 @@ impl QuantizedLinear {
pub fn new(inner: QMatMul, bias: Option<Tensor>) -> Self {
Self { inner, bias }
}
pub fn inner_dequantize(&self) -> Result<Tensor> {
Ok(self.inner.dequantize_f16()?)
}
}
impl Module for QuantizedLinear {
src/models/mod.rs
+20 -3
View File
@@ -22,7 +22,7 @@ pub mod w2v_bert_2_0;
use aha_openai_dive::v1::resources::chat::{
ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse,
};
use anyhow::Result;
use anyhow::{Result, anyhow};
use rocket::futures::Stream;
use crate::models::{
@@ -54,6 +54,8 @@ pub enum WhichModel {
Qwen3_5_4B,
#[value(name = "qwen3.5-9b", hide = true)]
Qwen3_5_9B,
#[value(name = "qwen3.5-gguf", hide = true)]
Qwen3_5Gguf,
#[value(name = "qwen3asr-0.6b", hide = true)]
Qwen3ASR0_6B,
#[value(name = "qwen3asr-1.7b", hide = true)]
@@ -98,6 +100,7 @@ impl WhichModel {
WhichModel::Qwen3_5_2B => "Qwen/Qwen3.5-2B",
WhichModel::Qwen3_5_4B => "Qwen/Qwen3.5-4B",
WhichModel::Qwen3_5_9B => "Qwen/Qwen3.5-9B",
WhichModel::Qwen3_5Gguf => "GGUF",
WhichModel::Qwen3ASR0_6B => "Qwen/Qwen3-ASR-0.6B",
WhichModel::Qwen3ASR1_7B => "Qwen/Qwen3-ASR-1.7B",
WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct",
@@ -130,7 +133,8 @@ impl WhichModel {
| WhichModel::Qwen3_5_0_8B
| WhichModel::Qwen3_5_2B
| WhichModel::Qwen3_5_4B
| WhichModel::Qwen3_5_9B => "vlm",
| WhichModel::Qwen3_5_9B
| WhichModel::Qwen3_5Gguf => "vlm",
// OCR models
WhichModel::DeepSeekOCR
| WhichModel::HunyuanOCR
@@ -229,7 +233,12 @@ impl<'a> GenerateModel for ModelInstance<'a> {
}
}
pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_>> {
pub fn load_model<'a>(
model_type: WhichModel,
path: &str,
gguf: Option<&str>,
mmproj: Option<&str>,
) -> Result<ModelInstance<'a>> {
let model = match model_type {
WhichModel::MiniCPM4_0_5B => {
let model = MiniCPMGenerateModel::init(path, None, None)?;
@@ -263,6 +272,14 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_
let model = Qwen3_5GenerateModel::init(path, None, None)?;
ModelInstance::Qwen3_5(model)
}
WhichModel::Qwen3_5Gguf => {
if gguf.is_none() {
return Err(anyhow!("Qwen3_5Gguf gguf model path is required"));
}
let gguf = gguf.unwrap();
let model = Qwen3_5GenerateModel::init_from_gguf(gguf, mmproj, None)?;
ModelInstance::Qwen3_5(model)
}
WhichModel::Qwen3ASR0_6B => {
let model = Qwen3AsrGenerateModel::init(path, None, None)?;
ModelInstance::Qwen3ASR(model)
src/models/qwen3_5/generate.rs
+6 -4
View File
@@ -102,7 +102,9 @@ impl<'a> Qwen3_5GenerateModel<'a> {
(None, None)
};
// let eos_token_id = gguf.get_matedata("tokenizer.ggml.eos_token_id")?.to_u32()?;
let eos_token_id = model_gguf
.get_matedata("tokenizer.ggml.eos_token_id")?
.to_u32()?;
let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut model_gguf, mmproj_gguf.as_mut(), &device)?;
let stem = std::path::Path::new(model_file)
.file_stem() // 获取文件名主干（不含扩展名）
@@ -114,7 +116,8 @@ impl<'a> Qwen3_5GenerateModel<'a> {
pre_processor,
qwen3_5,
device,
eos_token_id: 248044,
// eos_token_id: 248044,
eos_token_id,
model_name: stem.to_string(),
repeat_penalty: 1.1,
repeat_last_n: 64,
@@ -125,7 +128,7 @@ impl<'a> Qwen3_5GenerateModel<'a> {
impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
let seed = mes.seed.unwrap_or(32768) as u64;
let temperature = mes.temperature.unwrap_or(0.6);
let temperature = mes.temperature.unwrap_or(0.4);
let top_p = mes.top_p.unwrap_or(0.95);
let mut logit_processor =
get_logit_processor(temperature.into(), top_p.into(), Some(20), seed);
@@ -144,7 +147,6 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
} else {
(mes_render, None, None, None, None)
};
// let input = self.pre_processor.process_info(&mes, &mes_render)?;
let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?;
let mut seq_len = input_ids.dim(1)?;
let prompt_tokens = seq_len as u32;
src/models/qwen3_5/model.rs
+5 -6
View File
@@ -750,7 +750,7 @@ impl Qwen3_5Attention {
}
pub fn clear_kv_cache(&mut self) {
self.kv_cache = None
self.kv_cache = None;
}
}
@@ -1024,7 +1024,6 @@ impl Qwen3_5TextModel {
// i += 1;
}
xs = self.norm.forward(&xs)?;
// println!("norm : {}", xs);
Ok(xs)
}
@@ -1348,7 +1347,9 @@ impl Qwen3_5Model {
video_grid_thw: Option<&Tensor>,
seqlen_offset: usize,
) -> Result<Tensor> {
let position_ids = if let Some(rope_deltas) = &self.rope_deltas {
let position_ids = if let Some(rope_deltas) = &self.rope_deltas
&& seqlen_offset != 0
{
let (bs, seq_len, _) = inputs_embeds.dims3()?;
Tensor::arange(
seqlen_offset as i64,
@@ -1383,12 +1384,12 @@ impl Qwen3_5Model {
seqlen_offset: usize,
) -> Result<Tensor> {
let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?;
// println!("embed_tokens: {}", inputs_embeds);
if let Some(pixel_values) = pixel_values
&& let Some(image_grid_thw) = image_grid_thw
&& let Some(visual) = self.visual.as_ref()
{
let (image_embeds, _) = visual.forward(pixel_values, image_grid_thw)?;
// println!("image_embeds: {}", image_embeds);
let vision_mask = get_equal_mask(input_ids, self.image_token_id)?;
let n_image_tokens = vision_mask.sum_all()?.to_scalar::<u32>()?;
if n_image_tokens as usize != image_embeds.dim(0)? {
@@ -1429,9 +1430,7 @@ impl Qwen3_5Model {
let outputs = self.language_model.forward(&inputs_embeds, &position_ids)?;
let seq_len = outputs.dim(1)?;
let hidden_state = outputs.narrow(1, seq_len - 1, 1)?;
// println!("narrow 1 : {}", hidden_state);
let logits = self.lm_head.forward(&hidden_state)?;
// println!("logits : {}", logits);
Ok(logits)
}
src/models/qwen3vl/model.rs
+23 -6
View File
@@ -95,8 +95,9 @@ impl Qwen3VLVisionPatchEmbed {
pub fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
// hidden_states shape: (grid_t*grid_h*grid_w, c*temporal_patch_size*patch_size*patch_size)
// ((), 1536) matmul (1536, 1024) -> ((), 1024)
let hidden_states = hidden_states.matmul(&self.conv3d_weight)?;
let hidden_states = hidden_states.broadcast_add(&self.conv3d_bias)?;
let dtype = hidden_states.dtype();
let hidden_states = hidden_states.matmul(&self.conv3d_weight.to_dtype(dtype)?)?;
let hidden_states = hidden_states.broadcast_add(&self.conv3d_bias.to_dtype(dtype)?)?;
Ok(hidden_states)
}
}
@@ -169,7 +170,12 @@ impl Qwen3VLVisionPatchMerger {
} else {
xs.clone()
};
let xs = self.norm.forward(&xs)?.reshape(((), self.hidden_size))?;
let orig_dtype = xs.dtype();
let xs = self
.norm
.forward(&xs.to_dtype(self.norm.weight().dtype())?)?
.reshape(((), self.hidden_size))?;
let xs = xs.to_dtype(orig_dtype)?;
let xs = self
.linear_fc2
.forward(&self.act_fn.forward(&self.linear_fc1.forward(&xs)?)?)?;
@@ -343,12 +349,21 @@ impl Qwen3VLVisionBlock {
cos: &Tensor,
sin: &Tensor,
) -> Result<Tensor> {
let orig_dtype = xs.dtype();
let residual = xs.clone();
let xs = self.norm1.forward(xs)?;
let xs = self
.norm1
.forward(&xs.to_dtype(self.norm1.weight().dtype())?)?;
let xs = xs.to_dtype(orig_dtype)?;
let xs = self.attn.forward(&xs, cos, sin, cu_seqlens)?;
let xs = (residual + xs)?;
let residual = xs.clone();
let xs = self.mlp.forward(&self.norm2.forward(&xs)?)?;
let xs = self.mlp.forward(
&self
.norm2
.forward(&xs.to_dtype(self.norm2.weight().dtype())?)?
.to_dtype(orig_dtype)?,
)?;
let xs = (residual + xs)?;
Ok(xs)
}
@@ -679,7 +694,9 @@ impl Qwen3VLVisionModel {
grid_thw: &Tensor,
) -> Result<(Tensor, Vec<Tensor>)> {
let hidden_states = self.patch_embed.forward(hidden_states)?;
let pos_embeds = self.fast_pos_embed_interpolate(grid_thw)?;
let pos_embeds = self
.fast_pos_embed_interpolate(grid_thw)?
.to_dtype(hidden_states.dtype())?;
let hidden_states = hidden_states.broadcast_add(&pos_embeds)?;
let rotary_pos_emb = self.rot_pos_emb(grid_thw)?;
let seq_len = hidden_states.dim(0)?;
src/position_embed/rope.rs
+2
View File
@@ -520,6 +520,7 @@ impl Qwen3VLTextRotaryEmbedding {
let position_ids_expanded = position_ids
.unsqueeze(D::Minus2)?
.to_dtype(DType::F32)?
// .to_dtype(dtype)?
.contiguous()?;
// inv_freq Vec<f32> -> Tensor(1, 1, head_dim / 2, 1) -> (3, bs, head_dim / 2, 1)
let inv_freq_expanded = Tensor::from_vec(
@@ -529,6 +530,7 @@ impl Qwen3VLTextRotaryEmbedding {
)?
.broadcast_as((3, position_ids.dim(1)?, self.inv_freq.len(), 1))?
.to_dtype(DType::F32)?
// .to_dtype(dtype)?
.contiguous()?;
// (3, bs, head_dim / 2, 1) matmul (3, bs, 1, position)