tckm.yan/aha
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add Qwen3.5 mmproj gguf

Browse Source
This commit is contained in:
jhqxxx
2026-03-16 14:50:59 +08:00
parent 9c9275b767
commit 80d36b5308
14 changed files with 575 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_gguf_qwen3_5.rs
+40 -15
View File
@@ -5,24 +5,40 @@ use aha::{
models::{GenerateModel, qwen3_5::generate::Qwen3_5GenerateModel},
};
use anyhow::Result;
// use candle_core::{Device, quantized::gguf_file};
// use candle_core::{DType, Device, quantized::gguf_file};
#[test]
fn gguf_test() -> Result<()> {
// cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture
// let path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题
// let path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf";
let path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf";
// let mut file = std::fs::File::open(path)?;
// let model = gguf_file::Content::read(&mut file)?;
// println!("group_count: {:?}", model.metadata.get("qwen35.ssm.group_count"));
// println!("time_step_rank: {:?}", model.metadata.get("qwen35.ssm.time_step_rank"));
// println!("state_size: {:?}", model.metadata.get("qwen35.ssm.state_size"));
// RUST_BACKTRACE=1 cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture
// let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题
// let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/mmproj-F16.gguf";
// let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf";
let model_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf";
let mmproj_path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/mmproj-F16.gguf";
// let mut model_file = std::fs::File::open(model_path)?;
// let model = gguf_file::Content::read(&mut model_file)?;
// for (key, value) in model.metadata {
// if key.contains("tokenizer") {
// if key.contains("tokeni") {
// continue;
// }
// println!("{key}: {:#?}", value);
// }
// let mut mmproj_file = std::fs::File::open(mmproj_path)?;
// let mmproj = gguf_file::Content::read(&mut mmproj_file)?;
// println!("model: {:#?}", mmproj.tensor_infos.keys());
// println!("group_count: {:?}", model.metadata.get("qwen35.ssm.group_count"));
// println!("time_step_rank: {:?}", model.metadata.get("qwen35.ssm.time_step_rank"));
// println!("state_size: {:?}", model.metadata.get("qwen35.ssm.state_size"));
// for (key, value) in mmproj.metadata {
// println!("{key}: {:#?}", value);
// }
// let device = Device::new_cuda(0)?;
// let mut mmproj_gguf = Gguf::new(mmproj, mmproj_file, device.clone());
// let weight = mmproj_gguf.get_dequantized("v.position_embd.weight")?;
// println!("weight: {:?}", weight);
// let conv3d_weight_1 = mmproj_gguf.get_dequantized("v.patch_embd.weight.1")?;
// println!("conv3d_weight_1: {}", conv3d_weight_1);
// let conv3d_bias = mmproj_gguf.get_dequantized("v.patch_embd.bias")?;
// println!("conv3d_bias: {}", conv3d_bias);
// println!("model: {:?}", model.magic);
// println!("generat.type: {:#?}", model.metadata.keys());
// println!("tokenizer.ggml.eos_token_id: {:#?}", model.metadata.get("tokenizer.ggml.eos_token_id"));
@@ -33,19 +49,28 @@ fn gguf_test() -> Result<()> {
"messages": [
{
"role": "user",
"content": [
"content": [
{
"type": "text",
"text": "你如何看待AI"
"type": "image",
"image_url":
{
"url": "file://./assets/img/ocr_test1.png"
}
},
{
"type": "text",
"text": "请分析图片并提取所有可见文本内容，按从左到右、从上到下的布局，返回纯文本"
}
]
}
]
}
"#;
let mes: ChatCompletionParameters = serde_json::from_str(message)?;
let i_start = Instant::now();
let mut gguf_qwen3_5 = Qwen3_5GenerateModel::init_from_gguf(path, None, None)?;
let mut gguf_qwen3_5 =
Qwen3_5GenerateModel::init_from_gguf(model_path, mmproj_path.into(), None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
tests/test_qwen3_5.rs
+2 -2
View File
@@ -24,12 +24,12 @@ fn qwen3_5_generate() -> Result<()> {
"type": "image",
"image_url":
{
"url": "file:///home/jhq/Downloads/gougou1.jpg"
"url": "file://./assets/img/ocr_test1.png"
}
},
{
"type": "text",
"text": "描述这张图片."
"text": "OCR"
}
]
}
tests/test_qwen3vl.rs
+7 -7
View File
@@ -29,7 +29,7 @@ fn qwen3vl_thinking_generate() -> Result<()> {
},
{
"type": "text",
"text": "请分析图片并提取所有可见文本内容，按从左到右、从上到下的布局，返回纯文本"
"text": "OCR"
}
]
}
@@ -59,7 +59,7 @@ fn qwen3vl_thinking_generate() -> Result<()> {
#[test]
fn qwen3vl_generate() -> Result<()> {
// test with cuda: RUST_BACKTRACE=1 cargo test -F cuda,ffmpeg --test test_qwen3vl qwen3vl_generate -r -- --nocapture
// test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_qwen3vl qwen3vl_generate -r -- --nocapture
let save_dir =
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
@@ -73,15 +73,15 @@ fn qwen3vl_generate() -> Result<()> {
"role": "user",
"content": [
{
"type": "video",
"video_url":
"type": "image",
"image_url":
{
"url": "./assets/video/video_test.mp4"
"url": "file://./assets/img/ocr_test1.png"
}
},
},
{
"type": "text",
"text": "视频中发生了什么？"
"text": "OCR"
}
]
}