add Qwen3.5 model

src/api.rs

@@ -235,6 +235,10 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str {
         WhichModel::Qwen2_5vl3B => "qwen2.5vl-3b",
         WhichModel::Qwen2_5vl7B => "qwen2.5vl-7b",
         WhichModel::Qwen3_0_6B => "qwen3-0.6b",
+        WhichModel::Qwen3_5_0_8B => "qwen3.5-0.8b",
+        WhichModel::Qwen3_5_2B => "qwen3.5-2b",
+        WhichModel::Qwen3_5_4B => "qwen3.5-4b",
+        WhichModel::Qwen3_5_9B => "qwen3.5-9b",
         WhichModel::Qwen3ASR0_6B => "qwen3asr-0.6b",
         WhichModel::Qwen3ASR1_7B => "qwen3asr-1.7b",
         WhichModel::Qwen3vl2B => "qwen3vl-2b",
@@ -262,6 +266,10 @@ fn which_model_to_owner(which_model: WhichModel) -> &'static str {
         | WhichModel::Qwen3vl4B
         | WhichModel::Qwen3vl8B
         | WhichModel::Qwen3vl32B => "Qwen",
+        WhichModel::Qwen3_5_0_8B
+        | WhichModel::Qwen3_5_2B
+        | WhichModel::Qwen3_5_4B
+        | WhichModel::Qwen3_5_9B => "Qwen",
         WhichModel::DeepSeekOCR => "deepseek-ai",
         WhichModel::HunyuanOCR => "Tencent-Hunyuan",
         WhichModel::PaddleOCRVL => "PaddlePaddle",

src/chat_template/mod.rs

@@ -131,4 +131,25 @@ impl<'a> ChatTemplate<'a> {
             .map_err(|e| anyhow!(format!("render template error {}", e)))?;
         Ok(message_str)
     }
+
+    pub fn apply_chat_temp_think(
+        &self,
+        messages: &ChatCompletionParameters,
+        enable_thinking: Option<bool>,
+    ) -> Result<String> {
+        let context = context! {
+            messages => &messages.messages,
+            tools => &messages.tools.as_ref(),
+            add_generation_prompt => true,
+            enable_thinking => enable_thinking,
+        };
+        let template = self
+            .env
+            .get_template("chat")
+            .map_err(|e| anyhow!(format!("render template error {}", e)))?;
+        let message_str = template
+            .render(context)
+            .map_err(|e| anyhow!(format!("render template error {}", e)))?;
+        Ok(message_str)
+    }
 }

src/exec/mod.rs

@@ -11,6 +11,7 @@ pub mod minicpm4;
 pub mod paddleocr_vl;
 pub mod qwen2_5vl;
 pub mod qwen3;
+pub mod qwen3_5;
 pub mod qwen3_asr;
 pub mod qwen3vl;
 pub mod rmbg2_0;

src/exec/qwen3_5.rs

@@ -0,0 +1,103 @@
+//! Qwen3.5 exec implementation for CLI `run` subcommand
+
+use std::time::Instant;
+
+use anyhow::Result;
+
+use crate::exec::ExecModel;
+use crate::models::GenerateModel;
+use crate::models::qwen3_5::generate::Qwen3_5GenerateModel;
+use crate::utils::get_file_path;
+
+pub struct Qwen3_5Exec;
+
+impl ExecModel for Qwen3_5Exec {
+    fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
+        let input_text = &input[0];
+        let target_text = if input_text.starts_with("file://") {
+            let path = get_file_path(input_text)?;
+            std::fs::read_to_string(path)?
+        } else {
+            input_text.clone()
+        };
+
+        let i_start = Instant::now();
+        let mut model = Qwen3_5GenerateModel::init(weight_path, None, None)?;
+        let i_duration = i_start.elapsed();
+        println!("Time elapsed in load model is: {:?}", i_duration);
+        let url = &input[1];
+        let input_url = if url.starts_with("http://")
+            || url.starts_with("https://")
+            || url.starts_with("file://")
+        {
+            url.clone()
+        } else {
+            format!("file://{}", url)
+        };
+        let message = if input_url.ends_with("mp4") {
+            format!(
+                r#"{{
+            "model": "qwen3.5",
+            "messages": [
+                {{
+                    "role": "user",
+                    "content": [
+                        {{
+                            "type": "video",
+                            "video_url": 
+                            {{
+                                "url": "{}"
+                            }}
+                        }},
+                        {{
+                            "type": "text", 
+                            "text": "{}"
+                        }}
+                    ]
+                }}
+            ]
+        }}"#,
+                input_url, target_text
+            )
+        } else {
+            format!(
+                r#"{{
+            "model": "qwen2.5",
+            "messages": [
+                {{
+                    "role": "user",
+                    "content": [
+                        {{
+                            "type": "image",
+                            "image_url": {{
+                                "url": "{}"
+                            }}
+                        }},
+                        {{
+                            "type": "text", 
+                            "text": "{}"
+                        }}
+                    ]
+                }}
+            ]
+        }}"#,
+                input_url, target_text
+            )
+        };
+        let mes = serde_json::from_str(&message)?;
+
+        let i_start = Instant::now();
+        let result = model.generate(mes)?;
+        let i_duration = i_start.elapsed();
+        println!("Time elapsed in generate is: {:?}", i_duration);
+
+        println!("Result: {:?}", result);
+
+        if let Some(out) = output {
+            std::fs::write(out, format!("{:?}", result))?;
+            println!("Output saved to: {}", out);
+        }
+
+        Ok(())
+    }
+}

src/exec/qwen3vl.rs

@@ -61,7 +61,7 @@ impl ExecModel for Qwen3vlExec {
         } else {
             format!(
                 r#"{{
-            "model": "qwen2.5vl",
+            "model": "qwen3vl",
             "messages": [
                 {{
                     "role": "user",

src/lib.rs

@@ -5,3 +5,5 @@ pub mod position_embed;
 pub mod process;
 pub mod tokenizer;
 pub mod utils;
+
+pub use aha_openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatCompletionResponse};

src/main.rs

@@ -201,6 +201,10 @@ fn run_list(args: ListArgs) -> anyhow::Result<()> {
         WhichModel::Qwen2_5vl3B,
         WhichModel::Qwen2_5vl7B,
         WhichModel::Qwen3_0_6B,
+        WhichModel::Qwen3_5_0_8B,
+        WhichModel::Qwen3_5_2B,
+        WhichModel::Qwen3_5_4B,
+        WhichModel::Qwen3_5_9B,
         WhichModel::Qwen3ASR0_6B,
         WhichModel::Qwen3ASR1_7B,
         WhichModel::Qwen3vl2B,
@@ -390,6 +394,22 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
             use aha::exec::qwen3::Qwen3Exec;
             Qwen3Exec::run(&input, output.as_deref(), &weight_path)?;
         }
+        WhichModel::Qwen3_5_0_8B => {
+            use aha::exec::qwen3_5::Qwen3_5Exec;
+            Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
+        }
+        WhichModel::Qwen3_5_2B => {
+            use aha::exec::qwen3_5::Qwen3_5Exec;
+            Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
+        }
+        WhichModel::Qwen3_5_4B => {
+            use aha::exec::qwen3_5::Qwen3_5Exec;
+            Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
+        }
+        WhichModel::Qwen3_5_9B => {
+            use aha::exec::qwen3_5::Qwen3_5Exec;
+            Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
+        }
         WhichModel::Qwen3ASR0_6B => {
             use aha::exec::qwen3_asr::Qwen3ASRExec;
             Qwen3ASRExec::run(&input, output.as_deref(), &weight_path)?;

src/models/common/mod.rs

@@ -1,4 +1,4 @@
-use anyhow::{Result, anyhow};
+use anyhow::Result;
 use candle_core::{D, IndexOp, Tensor};
 use candle_nn::{
     Activation, BatchNorm, BatchNormConfig, Conv1d, Conv1dConfig, Conv2d, Conv2dConfig,
@@ -6,7 +6,6 @@ use candle_nn::{
     ModuleT, RmsNorm, VarBuilder, batch_norm, conv1d, conv1d_no_bias, conv2d, conv2d_no_bias,
     embedding, layer_norm, linear_b, linear_no_bias, ops::sigmoid, rms_norm,
 };
-use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
 
 use crate::{
     position_embed::rope::{RoPE, apply_rotary_pos_emb, apply_rotary_pos_emb_roformer},
@@ -971,49 +970,6 @@ impl LlamaForCausalLM {
     }
 }
 
-pub fn conv1d_group_parallel(xs: &Tensor, conv1d: &Conv1d) -> Result<Tensor> {
-    let groups = conv1d.config().groups;
-    let xs = if groups == 1 {
-        xs.conv1d_with_algo(
-            conv1d.weight(),
-            conv1d.config().padding,
-            conv1d.config().stride,
-            conv1d.config().dilation,
-            groups,
-            conv1d.config().cudnn_fwd_algo,
-        )?
-    } else {
-        let blocks = xs.chunk(groups, 1)?;
-        let kernel = conv1d.weight().chunk(groups, 0)?;
-        let blocks = blocks
-            // .iter()
-            .par_iter()
-            .zip(&kernel)
-            .map(|(block, kernel)| {
-                block
-                    .conv1d_with_algo(
-                        kernel,
-                        conv1d.config().padding,
-                        conv1d.config().stride,
-                        conv1d.config().dilation,
-                        1,
-                        conv1d.config().cudnn_fwd_algo,
-                    )
-                    .map_err(|e| anyhow!(format!("tensor conv1d_with_algo error:{}", e)))
-            })
-            .collect::<Result<Vec<Tensor>>>()?;
-        Tensor::cat(&blocks, 1)?
-    };
-    match conv1d.bias() {
-        None => Ok(xs),
-        Some(bias) => {
-            let b = bias.dims1()?;
-            let bias = bias.reshape((1, b, 1))?;
-            Ok(xs.broadcast_add(&bias)?)
-        }
-    }
-}
-
 pub struct GLU {
     dim: usize,
 }
@@ -1194,3 +1150,45 @@ pub fn mish(xs: &Tensor) -> Result<Tensor> {
     let xs = xs.mul(&tanh)?;
     Ok(xs)
 }
+
+pub fn softplus(xs: &Tensor) -> Result<Tensor> {
+    // ln(1 + exp(x))
+    Ok((xs.exp()? + 1.0)?.log()?)
+}
+
+pub fn softplus_stable(xs: &Tensor) -> Result<Tensor> {
+    // max(x, 0) + ln(1 + exp(-abs(x)))
+    let zero = Tensor::zeros_like(xs)?;
+    let x_max_0 = xs.maximum(&zero)?;
+    Ok((xs.abs()?.neg()?.exp()? + 1.0)?.log()?.add(&x_max_0)?)
+}
+
+// refer to https://github.com/huggingface/candle/issues/3389
+pub fn conv1d_depthwise(input: &Tensor, weight: &Tensor, bias: Option<&Tensor>) -> Result<Tensor> {
+    // group = dim, stride= 1
+    // input: (bs, dim, len)
+    // weight: (dim, 1, k) -> (dim, k)
+    // input already padding
+    let len_in = input.dim(2)?;
+    let weight = weight.squeeze(1)?;
+    let kernel_size = weight.dim(1)?;
+    // len_out = (len_in - k + 2p) / s + 1, p = 0, s = 1
+    let len_out = len_in - kernel_size + 1;
+    let mut out = input
+        .narrow(2, 0, len_out)?
+        .broadcast_mul(&weight.narrow(1, 0, 1)?.unsqueeze(0)?)?;
+    for k in 1..kernel_size {
+        out = (out
+            + input
+                .narrow(2, k, len_out)?
+                .broadcast_mul(&weight.narrow(1, k, 1)?.unsqueeze(0)?)?)?;
+    }
+    match bias {
+        None => Ok(out),
+        Some(bias) => {
+            let b = bias.dims1()?;
+            let bias = bias.reshape((1, b, 1))?;
+            Ok(out.broadcast_add(&bias)?)
+        }
+    }
+}

src/models/fun_asr_nano/model.rs

@@ -5,7 +5,8 @@ use candle_nn::{Conv1d, LayerNorm, Linear, Module, VarBuilder, linear, ops::soft
 use crate::{
     models::{
         common::{
-            NaiveAttention, TwoLinearMLP, eager_attention_forward, get_conv1d, get_layer_norm,
+            NaiveAttention, TwoLinearMLP, conv1d_depthwise, eager_attention_forward, get_conv1d,
+            get_layer_norm,
         },
         fun_asr_nano::config::FunASRNanoConfig,
         qwen3::{config::Qwen3Config, model::Qwen3Model},
@@ -85,7 +86,8 @@ impl MultiHeadedAttentionSANM {
         };
         let xs = inputs.transpose(1, 2)?;
         let xs = xs.pad_with_zeros(D::Minus1, self.left_padding, self.right_padding)?;
-        let xs = self.fsmn_block.forward(&xs)?;
+        // let xs = self.fsmn_block.forward(&xs)?;
+        let xs = conv1d_depthwise(&xs, self.fsmn_block.weight(), self.fsmn_block.bias())?;
         let xs = xs.transpose(1, 2)?;
         let mut xs = xs.add(&inputs)?;
         if let Some(mask) = mask {

src/models/mask_gct/model.rs

@@ -6,11 +6,10 @@ use candle_nn::{
 
 use crate::{
     models::{
-        common::{WNConv1d, get_conv1d, get_layer_norm},
+        common::{WNConv1d, conv1d_depthwise, get_conv1d, get_layer_norm},
         mask_gct::config::SemanticCodec,
     },
-    utils::interpolate::interpolate_nearest_1d,
-    utils::tensor_utils::l2_normalize,
+    utils::{interpolate::interpolate_nearest_1d, tensor_utils::l2_normalize},
 };
 
 pub struct ConvNeXtBlock {
@@ -43,7 +42,9 @@ impl ConvNeXtBlock {
     }
     pub fn forward(&self, xs: &Tensor) -> Result<Tensor> {
         let residual = xs.clone();
-        let xs = self.dwconv.forward(xs)?;
+        // let xs = self.dwconv.forward(xs)?;
+        let xs = xs.pad_with_zeros(D::Minus1, 3, 3)?;
+        let xs = conv1d_depthwise(&xs, self.dwconv.weight(), self.dwconv.bias())?;
         let xs = xs.transpose(1, 2)?;
         let xs = self.norm.forward(&xs)?;
         let xs = self.pwconv1.forward(&xs)?.gelu()?;

src/models/mod.rs

@@ -11,6 +11,7 @@ pub mod minicpm4;
 pub mod paddleocr_vl;
 pub mod qwen2_5vl;
 pub mod qwen3;
+pub mod qwen3_5;
 pub mod qwen3_asr;
 pub mod qwen3vl;
 pub mod rmbg2_0;
@@ -29,9 +30,9 @@ use crate::models::{
     glm_asr_nano::generate::GlmAsrNanoGenerateModel,
     hunyuan_ocr::generate::HunyuanOCRGenerateModel, minicpm4::generate::MiniCPMGenerateModel,
     paddleocr_vl::generate::PaddleOCRVLGenerateModel, qwen2_5vl::generate::Qwen2_5VLGenerateModel,
-    qwen3::generate::Qwen3GenerateModel, qwen3_asr::generate::Qwen3AsrGenerateModel,
-    qwen3vl::generate::Qwen3VLGenerateModel, rmbg2_0::generate::RMBG2_0Model,
-    voxcpm::generate::VoxCPMGenerate,
+    qwen3::generate::Qwen3GenerateModel, qwen3_5::generate::Qwen3_5GenerateModel,
+    qwen3_asr::generate::Qwen3AsrGenerateModel, qwen3vl::generate::Qwen3VLGenerateModel,
+    rmbg2_0::generate::RMBG2_0Model, voxcpm::generate::VoxCPMGenerate,
 };
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
@@ -44,6 +45,14 @@ pub enum WhichModel {
     Qwen2_5vl7B,
     #[value(name = "qwen3-0.6b", hide = true)]
     Qwen3_0_6B,
+    #[value(name = "qwen3.5-0.8b", hide = true)]
+    Qwen3_5_0_8B,
+    #[value(name = "qwen3.5-2b", hide = true)]
+    Qwen3_5_2B,
+    #[value(name = "qwen3.5-4b", hide = true)]
+    Qwen3_5_4B,
+    #[value(name = "qwen3.5-9b", hide = true)]
+    Qwen3_5_9B,
     #[value(name = "qwen3asr-0.6b", hide = true)]
     Qwen3ASR0_6B,
     #[value(name = "qwen3asr-1.7b", hide = true)]
@@ -82,6 +91,10 @@ impl WhichModel {
             WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct",
             WhichModel::Qwen2_5vl7B => "Qwen/Qwen2.5-VL-7B-Instruct",
             WhichModel::Qwen3_0_6B => "Qwen/Qwen3-0.6B",
+            WhichModel::Qwen3_5_0_8B => "Qwen/Qwen3.5-0.8B",
+            WhichModel::Qwen3_5_2B => "Qwen/Qwen3.5-2B",
+            WhichModel::Qwen3_5_4B => "Qwen/Qwen3.5-4B",
+            WhichModel::Qwen3_5_9B => "Qwen/Qwen3.5-9B",
             WhichModel::Qwen3ASR0_6B => "Qwen/Qwen3-ASR-0.6B",
             WhichModel::Qwen3ASR1_7B => "Qwen/Qwen3-ASR-1.7B",
             WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct",
@@ -103,14 +116,17 @@ impl WhichModel {
     pub fn model_type(self) -> &'static str {
         match self {
             // LLM models
-            WhichModel::MiniCPM4_0_5B
-            | WhichModel::Qwen2_5vl3B
+            WhichModel::MiniCPM4_0_5B | WhichModel::Qwen3_0_6B => "llm",
+            WhichModel::Qwen2_5vl3B
             | WhichModel::Qwen2_5vl7B
-            | WhichModel::Qwen3_0_6B
             | WhichModel::Qwen3vl2B
             | WhichModel::Qwen3vl4B
             | WhichModel::Qwen3vl8B
-            | WhichModel::Qwen3vl32B => "llm",
+            | WhichModel::Qwen3vl32B
+            | WhichModel::Qwen3_5_0_8B
+            | WhichModel::Qwen3_5_2B
+            | WhichModel::Qwen3_5_4B
+            | WhichModel::Qwen3_5_9B => "vlm",
             // OCR models
             WhichModel::DeepSeekOCR | WhichModel::HunyuanOCR | WhichModel::PaddleOCRVL => "ocr",
             // ASR models
@@ -143,6 +159,7 @@ pub enum ModelInstance<'a> {
     MiniCPM4(MiniCPMGenerateModel<'a>),
     Qwen2_5VL(Qwen2_5VLGenerateModel<'a>),
     Qwen3(Qwen3GenerateModel<'a>),
+    Qwen3_5(Qwen3_5GenerateModel<'a>),
     Qwen3ASR(Qwen3AsrGenerateModel<'a>),
     Qwen3VL(Qwen3VLGenerateModel<'a>),
     DeepSeekOCR(DeepseekOCRGenerateModel),
@@ -160,6 +177,7 @@ impl<'a> GenerateModel for ModelInstance<'a> {
             ModelInstance::MiniCPM4(model) => model.generate(mes),
             ModelInstance::Qwen2_5VL(model) => model.generate(mes),
             ModelInstance::Qwen3(model) => model.generate(mes),
+            ModelInstance::Qwen3_5(model) => model.generate(mes),
             ModelInstance::Qwen3ASR(model) => model.generate(mes),
             ModelInstance::Qwen3VL(model) => model.generate(mes),
             ModelInstance::DeepSeekOCR(model) => model.generate(mes),
@@ -187,6 +205,7 @@ impl<'a> GenerateModel for ModelInstance<'a> {
             ModelInstance::MiniCPM4(model) => model.generate_stream(mes),
             ModelInstance::Qwen2_5VL(model) => model.generate_stream(mes),
             ModelInstance::Qwen3(model) => model.generate_stream(mes),
+            ModelInstance::Qwen3_5(model) => model.generate_stream(mes),
             ModelInstance::Qwen3VL(model) => model.generate_stream(mes),
             ModelInstance::Qwen3ASR(model) => model.generate_stream(mes),
             ModelInstance::DeepSeekOCR(model) => model.generate_stream(mes),
@@ -218,6 +237,22 @@ pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_
             let model = Qwen3GenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3(model)
         }
+        WhichModel::Qwen3_5_0_8B => {
+            let model = Qwen3_5GenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3_5(model)
+        }
+        WhichModel::Qwen3_5_2B => {
+            let model = Qwen3_5GenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3_5(model)
+        }
+        WhichModel::Qwen3_5_4B => {
+            let model = Qwen3_5GenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3_5(model)
+        }
+        WhichModel::Qwen3_5_9B => {
+            let model = Qwen3_5GenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3_5(model)
+        }
         WhichModel::Qwen3ASR0_6B => {
             let model = Qwen3AsrGenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3ASR(model)

src/models/qwen3_5/config.rs

@@ -0,0 +1,57 @@
+use candle_nn::Activation;
+
+use crate::models::qwen3vl::config::Qwen3VLVisionConfig;
+
+#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
+pub struct RopeParameters {
+    pub mrope_interleaved: bool,
+    pub mrope_section: Vec<usize>,
+    pub rope_type: String,
+    pub rope_theta: f32,
+    pub partial_rotary_factor: f32,
+}
+
+#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
+pub struct Qwen3_5TextConfig {
+    pub attention_bias: bool,
+    pub attention_dropout: f32,
+    pub attn_output_gate: bool,
+    pub dtype: String,
+    pub eos_token_id: u32,
+    pub full_attention_interval: usize,
+    pub head_dim: usize,
+    pub hidden_act: Activation,
+    pub hidden_size: usize,
+    pub initializer_range: f32,
+    pub intermediate_size: usize,
+    pub layer_types: Vec<String>,
+    pub linear_conv_kernel_dim: usize,
+    pub linear_key_head_dim: usize,
+    pub linear_num_key_heads: usize,
+    pub linear_num_value_heads: usize,
+    pub linear_value_head_dim: usize,
+    pub max_position_embeddings: usize,
+    pub mlp_only_layers: Vec<usize>,
+    pub mtp_num_hidden_layers: usize,
+    pub mtp_use_dedicated_embeddings: bool,
+    pub num_attention_heads: usize,
+    pub num_hidden_layers: usize,
+    pub num_key_value_heads: usize,
+    pub rms_norm_eps: f64,
+    pub tie_word_embeddings: bool,
+    pub use_cache: bool,
+    pub vocab_size: usize,
+    pub mamba_ssm_dtype: String,
+    pub rope_parameters: RopeParameters,
+}
+
+#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
+pub struct Qwen3_5Config {
+    pub image_token_id: u32,
+    pub text_config: Qwen3_5TextConfig,
+    pub tie_word_embeddings: bool,
+    pub video_token_id: u32,
+    pub vision_config: Qwen3VLVisionConfig,
+    pub vision_end_token_id: u32,
+    pub vision_start_token_id: u32,
+}

src/models/qwen3_5/generate.rs

@@ -0,0 +1,231 @@
+use aha_openai_dive::v1::resources::chat::{
+    ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse,
+};
+use anyhow::{Result, anyhow};
+use candle_core::{DType, Device, Tensor};
+use candle_nn::VarBuilder;
+use rocket::async_stream::stream;
+use rocket::futures::Stream;
+
+use crate::{
+    chat_template::ChatTemplate,
+    models::{
+        GenerateModel,
+        qwen3_5::{config::Qwen3_5Config, model::Qwen3_5Model},
+        qwen3vl::processor::Qwen3VLProcessor,
+    },
+    tokenizer::TokenizerModel,
+    utils::{
+        build_completion_chunk_response, build_completion_response, extract_metadata_value, find_type_files, get_device, get_dtype, get_logit_processor
+    },
+};
+
+pub struct Qwen3_5GenerateModel<'a> {
+    chat_template: ChatTemplate<'a>,
+    tokenizer: TokenizerModel,
+    pre_processor: Qwen3VLProcessor,
+    qwen3_5: Qwen3_5Model,
+    device: Device,
+    eos_token_id: u32,
+    model_name: String,
+}
+
+impl<'a> Qwen3_5GenerateModel<'a> {
+    pub fn init(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self> {
+        let chat_template = ChatTemplate::init(path)?;
+        let tokenizer = TokenizerModel::init(path)?;
+        let config_path = path.to_string() + "/config.json";
+        let cfg: Qwen3_5Config = serde_json::from_slice(&std::fs::read(config_path)?)?;
+        let device = get_device(device);
+        let cfg_dtype = cfg.text_config.dtype.as_str();
+        let dtype = get_dtype(dtype, cfg_dtype);
+        let pre_processor = Qwen3VLProcessor::new(path, &device, dtype)?;
+        let model_list = find_type_files(path, "safetensors")?;
+        let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, &device)? };
+        let eos_token_id = cfg.text_config.eos_token_id;
+        let qwen3_5 = Qwen3_5Model::new(vb, cfg)?;
+
+        Ok(Self {
+            chat_template,
+            tokenizer,
+            pre_processor,
+            qwen3_5,
+            device,
+            eos_token_id,
+            model_name: "qwen3.5".to_string(),
+        })
+    }
+}
+
+impl<'a> GenerateModel for Qwen3_5GenerateModel<'a> {
+    fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let enable_thinking = extract_metadata_value::<bool>(&mes.metadata, "enable_thinking");
+        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
+        let mes_render = self.chat_template.apply_chat_temp_think(&mes, enable_thinking)?;
+        let input = self.pre_processor.process_info(&mes, &mes_render)?;
+        let mut input_ids = self
+            .tokenizer
+            .text_encode(input.replace_text.clone(), &self.device)?;
+        let mut seq_len = input_ids.dim(1)?;
+        let mut seqlen_offset = 0;
+        let mut pixel_values = input.pixel_values.as_ref();
+        let image_grid_thw = input.image_grid_thw.as_ref();
+        let mut pixel_values_video = input.pixel_values_video.as_ref();
+        let video_grid_thw = input.video_grid_thw.as_ref();
+        let mut generate = Vec::new();
+        let sample_len = mes.max_tokens.unwrap_or(1024);
+        for _ in 0..sample_len {
+            let logits = self.qwen3_5.forward(
+                &input_ids,
+                pixel_values,
+                image_grid_thw,
+                pixel_values_video,
+                video_grid_thw,
+                seqlen_offset,
+            )?;
+            let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
+            let next_token = logit_processor.sample(&logits)?;
+            generate.push(next_token);
+            if next_token == self.eos_token_id {
+                break;
+            }
+            seqlen_offset += seq_len;
+            seq_len = 1;
+            input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+            pixel_values = None;
+            pixel_values_video = None;
+        }
+        let num_token = generate.len() as u32;
+        let res = self.tokenizer.token_decode(generate)?;
+        self.qwen3_5.clear_cache();
+        let response = build_completion_response(res, &self.model_name, Some(num_token));
+        Ok(response)
+    }
+
+    fn generate_stream(
+        &mut self,
+        mes: ChatCompletionParameters,
+    ) -> Result<
+        Box<
+            dyn Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>
+                + Send
+                + Unpin
+                + '_,
+        >,
+    > {
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
+        let enable_thinking = extract_metadata_value::<bool>(&mes.metadata, "enable_thinking");
+        let mes_render = self.chat_template.apply_chat_temp_think(&mes, enable_thinking)?;
+        let input = self.pre_processor.process_info(&mes, &mes_render)?;
+        let mut input_ids = self
+            .tokenizer
+            .text_encode(input.replace_text.clone(), &self.device)?;
+        let mut seq_len = input_ids.dim(1)?;
+        let mut seqlen_offset = 0;
+        let sample_len = mes.max_tokens.unwrap_or(1024);
+        let stream = stream! {
+            let mut error_tokens = Vec::new();
+            let mut pixel_values = input.pixel_values.as_ref();
+            let image_grid_thw = input.image_grid_thw.as_ref();
+            let mut pixel_values_video = input.pixel_values_video.as_ref();
+            let video_grid_thw = input.video_grid_thw.as_ref();
+            let mut tool_call_id = None;
+            let mut tool_call_content = String::new();
+            for _ in 0..sample_len {
+                let logits = self.qwen3_5.forward(
+                    &input_ids,
+                    pixel_values,
+                    image_grid_thw,
+                    pixel_values_video,
+                    video_grid_thw,
+                    seqlen_offset,
+                )?;
+                let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
+                let next_token = logit_processor.sample(&logits)?;
+                let mut decode_ids = Vec::new();
+                if !error_tokens.is_empty() {
+                    decode_ids.extend_from_slice(&error_tokens);
+                }
+                decode_ids.push(next_token);
+                let decoded_token = self.tokenizer.token_decode(decode_ids).map_err(|e| anyhow!(format!("stream decode error{e}")))?;
+                if decoded_token.contains("�") {
+                    error_tokens.push(next_token);
+                    if error_tokens.len() > 3 {
+                        error_tokens.clear();
+                    }
+                    seqlen_offset += seq_len;
+                    seq_len = 1;
+                    input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+                    pixel_values = None;
+                    pixel_values_video = None;
+                    continue;
+                }
+                error_tokens.clear();
+                // 处理特殊标记和工具调用
+                match decoded_token.as_str() {
+                    "<tool_call>" => {
+                        // 开始工具调用
+                        tool_call_id = Some(uuid::Uuid::new_v4().to_string());
+                        seqlen_offset += seq_len;
+                        seq_len = 1;
+                        input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+                        pixel_values = None;
+                        pixel_values_video = None;
+                        continue;
+                    }
+                    "</tool_call>" => {
+                        // 结束工具调用
+                        let chunk = build_completion_chunk_response(
+                            decoded_token,
+                            &self.model_name,
+                            tool_call_id.clone(),
+                            Some(tool_call_content.clone())
+                        );
+                        tool_call_id = None;
+                        tool_call_content = String::new();
+                        yield Ok(chunk);
+                    }
+                    _ => {
+                        if tool_call_id.is_some() {
+                            // 在工具调用过程中，收集工具调用内容
+                            tool_call_content.push_str(&decoded_token);
+                            seqlen_offset += seq_len;
+                            seq_len = 1;
+                            input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+                            pixel_values = None;
+                            pixel_values_video = None;
+                            continue;
+                        } else {
+                            // 正常文本输出
+                            let chunk = build_completion_chunk_response(
+                                decoded_token,
+                                &self.model_name,
+                                None,
+                                None
+                            );
+                            yield Ok(chunk);
+                        }
+                    }
+                }
+                if next_token == self.eos_token_id {
+                    break;
+                }
+                seqlen_offset += seq_len;
+                seq_len = 1;
+                input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+                pixel_values = None;
+                pixel_values_video = None;
+            }
+            self.qwen3_5.clear_cache();
+        };
+        Ok(Box::new(Box::pin(stream)))
+    }
+}

src/models/qwen3_5/mod.rs

@@ -0,0 +1,3 @@
+pub mod config;
+pub mod generate;
+pub mod model;

src/models/w2v_bert_2_0/model.rs

@@ -7,7 +7,10 @@ use candle_nn::{
 
 use crate::{
     models::{
-        common::{GLU, TwoLinearMLP, eager_attention_forward, get_conv1d, get_layer_norm},
+        common::{
+            GLU, TwoLinearMLP, conv1d_depthwise, eager_attention_forward, get_conv1d,
+            get_layer_norm,
+        },
         w2v_bert_2_0::config::W2VBert2_0Config,
     },
     position_embed::rope::{RoPE, apply_rotary_pos_emb},
@@ -309,7 +312,12 @@ impl Wav2Vec2BertConvolutionModule {
         // (batch, channel, dim)
         let xs = self.glu.forward(&xs)?;
         let xs = xs.pad_with_zeros(D::Minus1, self.conv_depthwise_kernel_size - 1, 0)?;
-        let xs = self.depthwise_conv.forward(&xs)?;
+        // let xs = self.depthwise_conv.forward(&xs)?;
+        let xs = conv1d_depthwise(
+            &xs,
+            self.depthwise_conv.weight(),
+            self.depthwise_conv.bias(),
+        )?;
         let xs = self
             .depthwise_layer_norm
             .forward(&xs.transpose(1, 2)?)?

src/position_embed/rope.rs

@@ -142,10 +142,11 @@ pub fn glm_asr_apply_rotary_pos_emb(
     let cos = cos.to_dtype(q.dtype())?;
     let sin = sin.to_dtype(q.dtype())?;
     let rotary_dim = cos.dim(D::Minus1)?;
+    let q_dim = q.dim(D::Minus1)?;
     let q_rot = q.narrow(D::Minus1, 0, rotary_dim)?;
-    let q_pass = q.narrow(D::Minus1, rotary_dim, rotary_dim)?;
+    let q_pass = q.narrow(D::Minus1, rotary_dim, q_dim - rotary_dim)?;
     let k_rot = k.narrow(D::Minus1, 0, rotary_dim)?;
-    let k_pass = k.narrow(D::Minus1, rotary_dim, rotary_dim)?;
+    let k_pass = k.narrow(D::Minus1, rotary_dim, q_dim - rotary_dim)?;
 
     let q_embed = q_rot
         .broadcast_mul(&cos)?

src/utils/tensor_utils.rs

@@ -554,7 +554,7 @@ pub fn l2_normalize(t: &Tensor, dim: usize) -> Result<Tensor> {
     if dim >= rank {
         return Err(anyhow!(format!("input dim {} must < rank {}", dim, rank)));
     }
-    let l2_norm = t.sqr()?.sum_keepdim(dim)?.sqrt()?;
+    let l2_norm = t.sqr()?.sum_keepdim(dim)?.affine(1.0, 1e-6)?.sqrt()?;
     Ok(t.broadcast_div(&l2_norm)?)
 }
 
@@ -650,3 +650,29 @@ pub fn cosine_similarity(query_vector: &Tensor, matrix: &Tensor) -> Result<Tenso
         .squeeze(D::Minus1)?;
     Ok(similarity)
 }
+
+pub fn repeat_interleave(t: &Tensor, repeats: usize, dim: usize) -> Result<Tensor> {
+    if repeats == 1 {
+        return Ok(t.clone());
+    }
+    let rank = t.rank();
+    if dim >= rank {
+        return Err(anyhow!(
+            "Dimension {} is out of range for tensor with {} dimensions",
+            dim,
+            rank
+        ));
+    }
+
+    let dims = t.dims();
+    let mut indices = Vec::with_capacity(dims[dim] * repeats);
+    for i in 0..dims[dim] {
+        for _ in 0..repeats {
+            indices.push(i as u32);
+        }
+    }
+
+    let indices_tensor = Tensor::from_vec(indices, (dims[dim] * repeats,), t.device())?;
+    let t = t.index_select(&indices_tensor, dim)?;
+    Ok(t)
+}