add qwen3.5 gguf

tests/test_gguf_qwen3_5.rs

@@ -1,18 +1,32 @@
-use aha::{chat::ChatCompletionParameters, gguf_models::qwen3_5::GgufQwen3_5};
+use std::time::Instant;
+
+use aha::{
+    chat::ChatCompletionParameters,
+    models::{GenerateModel, qwen3_5::generate::Qwen3_5GenerateModel},
+};
 use anyhow::Result;
-use candle_core::{Device, quantized::gguf_file};
+// use candle_core::{Device, quantized::gguf_file};
 #[test]
 fn gguf_test() -> Result<()> {
     // cargo test -r -F cuda --test test_gguf_qwen3_5 gguf_test -- --nocapture
+    // let path = "/home/jhq/.aha/Qwen/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q5_K_M.gguf"; // 有问题
+    // let path = "/home/jhq/.aha/Qwen/Qwen3.5-2B-GGUF/Qwen3.5-2B-Q6_K.gguf";
     let path = "/home/jhq/.aha/Qwen/Qwen3.5-0.8B-GGUF/Qwen3.5-0.8B-Q4_K_M.gguf";
-    let mut file = std::fs::File::open(path)?;
-    let model = gguf_file::Content::read(&mut file)?;
-    let device = Device::new_cuda(0)?;
+    // let mut file = std::fs::File::open(path)?;
+    // let model = gguf_file::Content::read(&mut file)?;
+    // println!("group_count: {:?}", model.metadata.get("qwen35.ssm.group_count"));
+    // println!("time_step_rank: {:?}", model.metadata.get("qwen35.ssm.time_step_rank"));
+    // println!("state_size: {:?}", model.metadata.get("qwen35.ssm.state_size"));
+    // for (key, value) in model.metadata {
+    //     if key.contains("tokenizer") {
+    //         continue;
+    //     }
+    //     println!("{key}: {:#?}", value);
+    // }
     // println!("model: {:?}", model.magic);
-    // println!("generat.type: {:#?}", model.metadata.keys()); 
-    // println!("tokenizer.ggml.model: {:#?}", model.metadata.get("tokenizer.ggml.model"));  // gpt2
-    // // println!("model: {:?}", model.tensor_infos);
-
+    // println!("generat.type: {:#?}", model.metadata.keys());
+    // println!("tokenizer.ggml.eos_token_id: {:#?}", model.metadata.get("tokenizer.ggml.eos_token_id"));
+    // println!("model: {:#?}", model.tensor_infos.keys());
     let message = r#"
     {
         "model": "qwen3.5",
@@ -22,26 +36,29 @@ fn gguf_test() -> Result<()> {
                 "content": [        
                     {
                         "type": "text", 
-                        "text": "你好啊"
+                        "text": "你如何看待AI"
                     }
                 ]
             }
         ]
     }
     "#;
-// render: <|im_start|>user
-// 你好啊<|im_end|>
-// <|im_start|>assistant
-// <think>
-
-// </think>
-
-
-// input_ids: [[248045,    846,    198, 109266,  98710, 248046,    198, 248045,  74455,    198,
-//   248068,    271, 248069,    271]]
-// Tensor[[1, 14], u32, cuda:0]
     let mes: ChatCompletionParameters = serde_json::from_str(message)?;
-    let mut gguf_qwen3_5 = GgufQwen3_5::new(&path, None)?;
-    let _ = gguf_qwen3_5.generate(mes)?;
+    let i_start = Instant::now();
+    let mut gguf_qwen3_5 = Qwen3_5GenerateModel::init_from_gguf(path, None, None)?;
+    let i_duration = i_start.elapsed();
+    println!("Time elapsed in load model is: {:?}", i_duration);
+
+    let i_start = Instant::now();
+    let res = gguf_qwen3_5.generate(mes)?;
+    let i_duration = i_start.elapsed();
+    println!("generate: \n {:?}", res);
+    if res.usage.is_some() {
+        let num_token = res.usage.as_ref().unwrap().total_tokens;
+        let duration_secs = i_duration.as_secs_f64();
+        let tps = num_token as f64 / duration_secs;
+        println!("Tokens per second (TPS): {:.2}", tps);
+    }
+    println!("Time elapsed in generate is: {:?}", i_duration);
     Ok(())
-}
+}

tests/test_qwen3_5.rs

@@ -22,7 +22,7 @@ fn qwen3_5_generate() -> Result<()> {
                 "content": [        
                     {
                         "type": "text", 
-                        "text": "你好啊"
+                        "text": "你好啊，你是谁"
                     }
                 ]
             }
@@ -31,12 +31,12 @@ fn qwen3_5_generate() -> Result<()> {
     "#;
     let mes: ChatCompletionParameters = serde_json::from_str(message)?;
     let i_start = Instant::now();
-    let mut qwen3vl = Qwen3_5GenerateModel::init(&model_path, None, None)?;
+    let mut qwen3_5 = Qwen3_5GenerateModel::init(&model_path, None, None)?;
     let i_duration = i_start.elapsed();
     println!("Time elapsed in load model is: {:?}", i_duration);
 
     let i_start = Instant::now();
-    let res = qwen3vl.generate(mes)?;
+    let res = qwen3_5.generate(mes)?;
     let i_duration = i_start.elapsed();
     println!("generate: \n {:?}", res);
     if res.usage.is_some() {