cli add about gguf param

src/models/common/gguf.rs

@@ -213,6 +213,10 @@ impl QuantizedLinear {
     pub fn new(inner: QMatMul, bias: Option<Tensor>) -> Self {
         Self { inner, bias }
     }
+
+    pub fn inner_dequantize(&self) -> Result<Tensor> {
+        Ok(self.inner.dequantize_f16()?)
+    }
 }
 
 impl Module for QuantizedLinear {

src/models/mod.rs

@@ -22,7 +22,7 @@ pub mod w2v_bert_2_0;
 use aha_openai_dive::v1::resources::chat::{
     ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse,
 };
-use anyhow::Result;
+use anyhow::{Result, anyhow};
 use rocket::futures::Stream;
 
 use crate::models::{
@@ -54,6 +54,8 @@ pub enum WhichModel {
     Qwen3_5_4B,
     #[value(name = "qwen3.5-9b", hide = true)]
     Qwen3_5_9B,
+    #[value(name = "qwen3.5-gguf", hide = true)]
+    Qwen3_5Gguf,
     #[value(name = "qwen3asr-0.6b", hide = true)]
     Qwen3ASR0_6B,
     #[value(name = "qwen3asr-1.7b", hide = true)]
@@ -98,6 +100,7 @@ impl WhichModel {
             WhichModel::Qwen3_5_2B => "Qwen/Qwen3.5-2B",
             WhichModel::Qwen3_5_4B => "Qwen/Qwen3.5-4B",
             WhichModel::Qwen3_5_9B => "Qwen/Qwen3.5-9B",
+            WhichModel::Qwen3_5Gguf => "GGUF",
             WhichModel::Qwen3ASR0_6B => "Qwen/Qwen3-ASR-0.6B",
             WhichModel::Qwen3ASR1_7B => "Qwen/Qwen3-ASR-1.7B",
             WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct",
@@ -130,7 +133,8 @@ impl WhichModel {
             | WhichModel::Qwen3_5_0_8B
             | WhichModel::Qwen3_5_2B
             | WhichModel::Qwen3_5_4B
-            | WhichModel::Qwen3_5_9B => "vlm",
+            | WhichModel::Qwen3_5_9B
+            | WhichModel::Qwen3_5Gguf => "vlm",
             // OCR models
             WhichModel::DeepSeekOCR
             | WhichModel::HunyuanOCR
@@ -229,7 +233,12 @@ impl<'a> GenerateModel for ModelInstance<'a> {
     }
 }
 
-pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_>> {
+pub fn load_model<'a>(
+    model_type: WhichModel,
+    path: &str,
+    gguf: Option<&str>,
+    mmproj: Option<&str>,
+) -> Result<ModelInstance<'a>> {
     let model = match model_type {
         WhichModel::MiniCPM4_0_5B => {
             let model = MiniCPMGenerateModel::init(path, None, None)?;
@@ -263,6 +272,14 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_
             let model = Qwen3_5GenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3_5(model)
         }
+        WhichModel::Qwen3_5Gguf => {
+            if gguf.is_none() {
+                return Err(anyhow!("Qwen3_5Gguf gguf model path is required"));
+            }
+            let gguf = gguf.unwrap();
+            let model = Qwen3_5GenerateModel::init_from_gguf(gguf, mmproj, None)?;
+            ModelInstance::Qwen3_5(model)
+        }
         WhichModel::Qwen3ASR0_6B => {
             let model = Qwen3AsrGenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3ASR(model)

src/models/qwen3_5/generate.rs

@@ -102,7 +102,9 @@ impl<'a> Qwen3_5GenerateModel<'a> {
             (None, None)
         };
 
-        // let eos_token_id = gguf.get_matedata("tokenizer.ggml.eos_token_id")?.to_u32()?;
+        let eos_token_id = model_gguf
+            .get_matedata("tokenizer.ggml.eos_token_id")?
+            .to_u32()?;
         let qwen3_5 = Qwen3_5Model::new_from_gguf(&mut model_gguf, mmproj_gguf.as_mut(), &device)?;
         let stem = std::path::Path::new(model_file)
             .file_stem() // 获取文件名主干（不含扩展名）
@@ -114,7 +116,8 @@ impl<'a> Qwen3_5GenerateModel<'a> {
             pre_processor,
             qwen3_5,
             device,
-            eos_token_id: 248044,
+            // eos_token_id: 248044,
+            eos_token_id,
             model_name: stem.to_string(),
             repeat_penalty: 1.1,
             repeat_last_n: 64,
@@ -125,7 +128,7 @@ impl<'a> Qwen3_5GenerateModel<'a> {
 impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
     fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
         let seed = mes.seed.unwrap_or(32768) as u64;
-        let temperature = mes.temperature.unwrap_or(0.6);
+        let temperature = mes.temperature.unwrap_or(0.4);
         let top_p = mes.top_p.unwrap_or(0.95);
         let mut logit_processor =
             get_logit_processor(temperature.into(), top_p.into(), Some(20), seed);
@@ -144,7 +147,6 @@ impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
             } else {
                 (mes_render, None, None, None, None)
             };
-        // let input = self.pre_processor.process_info(&mes, &mes_render)?;
         let mut input_ids = self.tokenizer.text_encode(mes_text, &self.device)?;
         let mut seq_len = input_ids.dim(1)?;
         let prompt_tokens = seq_len as u32;

src/models/qwen3_5/model.rs

@@ -750,7 +750,7 @@ impl Qwen3_5Attention {
     }
 
     pub fn clear_kv_cache(&mut self) {
-        self.kv_cache = None
+        self.kv_cache = None;
     }
 }
 
@@ -1024,7 +1024,6 @@ impl Qwen3_5TextModel {
             // i += 1;
         }
         xs = self.norm.forward(&xs)?;
-        // println!("norm : {}", xs);
         Ok(xs)
     }
 
@@ -1348,7 +1347,9 @@ impl Qwen3_5Model {
         video_grid_thw: Option<&Tensor>,
         seqlen_offset: usize,
     ) -> Result<Tensor> {
-        let position_ids = if let Some(rope_deltas) = &self.rope_deltas {
+        let position_ids = if let Some(rope_deltas) = &self.rope_deltas
+            && seqlen_offset != 0
+        {
             let (bs, seq_len, _) = inputs_embeds.dims3()?;
             Tensor::arange(
                 seqlen_offset as i64,
@@ -1383,12 +1384,12 @@ impl Qwen3_5Model {
         seqlen_offset: usize,
     ) -> Result<Tensor> {
         let mut inputs_embeds = self.language_model.embed_tokens.forward(input_ids)?;
-        // println!("embed_tokens: {}", inputs_embeds);
         if let Some(pixel_values) = pixel_values
             && let Some(image_grid_thw) = image_grid_thw
             && let Some(visual) = self.visual.as_ref()
         {
             let (image_embeds, _) = visual.forward(pixel_values, image_grid_thw)?;
+            // println!("image_embeds: {}", image_embeds);
             let vision_mask = get_equal_mask(input_ids, self.image_token_id)?;
             let n_image_tokens = vision_mask.sum_all()?.to_scalar::<u32>()?;
             if n_image_tokens as usize != image_embeds.dim(0)? {
@@ -1429,9 +1430,7 @@ impl Qwen3_5Model {
         let outputs = self.language_model.forward(&inputs_embeds, &position_ids)?;
         let seq_len = outputs.dim(1)?;
         let hidden_state = outputs.narrow(1, seq_len - 1, 1)?;
-        // println!("narrow 1 : {}", hidden_state);
         let logits = self.lm_head.forward(&hidden_state)?;
-        // println!("logits : {}", logits);
         Ok(logits)
     }
 

src/models/qwen3vl/model.rs

@@ -95,8 +95,9 @@ impl Qwen3VLVisionPatchEmbed {
     pub fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
         // hidden_states shape:  (grid_t*grid_h*grid_w, c*temporal_patch_size*patch_size*patch_size)
         // ((), 1536) matmul (1536, 1024) -> ((), 1024)
-        let hidden_states = hidden_states.matmul(&self.conv3d_weight)?;
-        let hidden_states = hidden_states.broadcast_add(&self.conv3d_bias)?;
+        let dtype = hidden_states.dtype();
+        let hidden_states = hidden_states.matmul(&self.conv3d_weight.to_dtype(dtype)?)?;
+        let hidden_states = hidden_states.broadcast_add(&self.conv3d_bias.to_dtype(dtype)?)?;
         Ok(hidden_states)
     }
 }
@@ -169,7 +170,12 @@ impl Qwen3VLVisionPatchMerger {
         } else {
             xs.clone()
         };
-        let xs = self.norm.forward(&xs)?.reshape(((), self.hidden_size))?;
+        let orig_dtype = xs.dtype();
+        let xs = self
+            .norm
+            .forward(&xs.to_dtype(self.norm.weight().dtype())?)?
+            .reshape(((), self.hidden_size))?;
+        let xs = xs.to_dtype(orig_dtype)?;
         let xs = self
             .linear_fc2
             .forward(&self.act_fn.forward(&self.linear_fc1.forward(&xs)?)?)?;
@@ -343,12 +349,21 @@ impl Qwen3VLVisionBlock {
         cos: &Tensor,
         sin: &Tensor,
     ) -> Result<Tensor> {
+        let orig_dtype = xs.dtype();
         let residual = xs.clone();
-        let xs = self.norm1.forward(xs)?;
+        let xs = self
+            .norm1
+            .forward(&xs.to_dtype(self.norm1.weight().dtype())?)?;
+        let xs = xs.to_dtype(orig_dtype)?;
         let xs = self.attn.forward(&xs, cos, sin, cu_seqlens)?;
         let xs = (residual + xs)?;
         let residual = xs.clone();
-        let xs = self.mlp.forward(&self.norm2.forward(&xs)?)?;
+        let xs = self.mlp.forward(
+            &self
+                .norm2
+                .forward(&xs.to_dtype(self.norm2.weight().dtype())?)?
+                .to_dtype(orig_dtype)?,
+        )?;
         let xs = (residual + xs)?;
         Ok(xs)
     }
@@ -679,7 +694,9 @@ impl Qwen3VLVisionModel {
         grid_thw: &Tensor,
     ) -> Result<(Tensor, Vec<Tensor>)> {
         let hidden_states = self.patch_embed.forward(hidden_states)?;
-        let pos_embeds = self.fast_pos_embed_interpolate(grid_thw)?;
+        let pos_embeds = self
+            .fast_pos_embed_interpolate(grid_thw)?
+            .to_dtype(hidden_states.dtype())?;
         let hidden_states = hidden_states.broadcast_add(&pos_embeds)?;
         let rotary_pos_emb = self.rot_pos_emb(grid_thw)?;
         let seq_len = hidden_states.dim(0)?;