merge main

src/models/mod.rs

@@ -34,37 +34,37 @@ use crate::models::{
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
 pub enum WhichModel {
-    #[value(name = "minicpm4-0.5b")]
+    #[value(name = "minicpm4-0.5b", hide = true)]
     MiniCPM4_0_5B,
-    #[value(name = "qwen2.5vl-3b")]
+    #[value(name = "qwen2.5vl-3b", hide = true)]
     Qwen2_5vl3B,
-    #[value(name = "qwen2.5vl-7b")]
+    #[value(name = "qwen2.5vl-7b", hide = true)]
     Qwen2_5vl7B,
-    #[value(name = "qwen3-0.6b")]
+    #[value(name = "qwen3-0.6b", hide = true)]
     Qwen3_0_6B,
-    #[value(name = "qwen3vl-2b")]
+    #[value(name = "qwen3vl-2b", hide = true)]
     Qwen3vl2B,
-    #[value(name = "qwen3vl-4b")]
+    #[value(name = "qwen3vl-4b", hide = true)]
     Qwen3vl4B,
-    #[value(name = "qwen3vl-8b")]
+    #[value(name = "qwen3vl-8b", hide = true)]
     Qwen3vl8B,
-    #[value(name = "qwen3vl-32b")]
+    #[value(name = "qwen3vl-32b", hide = true)]
     Qwen3vl32B,
-    #[value(name = "deepseek-ocr")]
+    #[value(name = "deepseek-ocr", hide = true)]
     DeepSeekOCR,
-    #[value(name = "hunyuan-ocr")]
+    #[value(name = "hunyuan-ocr", hide = true)]
     HunyuanOCR,
-    #[value(name = "paddleocr-vl")]
+    #[value(name = "paddleocr-vl", hide = true)]
     PaddleOCRVL,
     #[value(name = "rmbg2.0")]
     RMBG2_0,
-    #[value(name = "voxcpm")]
+    #[value(name = "voxcpm", hide = true)]
     VoxCPM,
-    #[value(name = "voxcpm1.5")]
+    #[value(name = "voxcpm1.5", hide = true)]
     VoxCPM1_5,
-    #[value(name = "glm-asr-nano-2512")]
+    #[value(name = "glm-asr-nano-2512", hide = true)]
     GlmASRNano2512,
-    #[value(name = "fun-asr-nano-2512")]
+    #[value(name = "fun-asr-nano-2512", hide = true)]
     FunASRNano2512,
 }
 

src/models/qwen3_asr/config.rs

@@ -0,0 +1,92 @@
+use serde::Deserialize;
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct FunASRNanoConfig {
+    pub audio_encoder_conf: AudioEncoderConf,
+    pub llm_conf: LlmConf,
+    pub audio_adaptor_conf: AudioAdaptorConf,
+    pub detach_ctc_decoder: bool,
+    pub ctc_decoder_conf: CtcDecoderConf,
+    pub ctc_weight: f64,
+    pub ctc_conf: CtcConf,
+    pub frontend_conf: FrontendConf,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct AudioEncoderConf {
+    pub output_size: usize,
+    pub attention_heads: usize,
+    pub linear_units: usize,
+    pub num_blocks: usize,
+    pub tp_blocks: usize,
+    pub dropout_rate: f64,
+    pub positional_dropout_rate: f64,
+    pub attention_dropout_rate: f64,
+    pub input_layer: String,
+    pub pos_enc_class: String,
+    pub normalize_before: bool,
+    pub kernel_size: usize,
+    pub sanm_shfit: usize,
+    pub selfattention_layer_type: String,
+    pub freeze: bool,
+    pub freeze_layer_num: i32,
+    pub feat_permute: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct LlmConf {
+    pub hub: String,
+    pub freeze: bool,
+    pub llm_dtype: String,
+    pub init_param_path: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct AudioAdaptorConf {
+    pub downsample_rate: usize,
+    pub use_low_frame_rate: bool,
+    pub ffn_dim: usize,
+    pub llm_dim: usize,
+    pub encoder_dim: usize,
+    pub n_layer: usize,
+    pub freeze: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct CtcDecoderConf {
+    pub downsample_rate: u32,
+    pub ffn_dim: u32,
+    pub llm_dim: u32,
+    pub encoder_dim: u32,
+    pub n_layer: u32,
+    pub freeze: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct CtcConf {
+    pub dropout_rate: f64,
+    pub ctc_type: String,
+    pub reduce: bool,
+    pub ignore_nan_grad: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct FrontendConf {
+    pub fs: usize,
+    pub window: String,
+    pub n_mels: usize,
+    pub frame_length: f32,
+    pub frame_shift: f32,
+    pub lfr_m: usize,
+    pub lfr_n: usize,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cmvn_file: Option<serde_yaml::Value>,
+}
+
+#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
+pub struct Qwen3ASRGenerationConfig {
+    pub do_sample: bool,
+    pub eos_token_id: Vec<usize>,
+    pub pad_token_id: usize,
+    pub temperature: f32,
+}

src/models/qwen3_asr/generate.rs

@@ -0,0 +1,212 @@
+use std::collections::HashMap;
+
+use aha_openai_dive::v1::resources::chat::{
+    ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse,
+};
+use anyhow::{Result, anyhow};
+use candle_core::{DType, Device, Tensor, pickle::read_all_with_key};
+use candle_nn::VarBuilder;
+use rocket::async_stream::stream;
+use rocket::futures::Stream;
+
+use crate::{
+    chat_template::ChatTemplate, models::{
+        GenerateModel,
+        fun_asr_nano::{
+            config::FunASRNanoConfig, model::FunAsrNanoModel,
+        },
+        qwen3::config::{Qwen3Config, Qwen3GenerationConfig}, qwen3_asr::{config::Qwen3ASRGenerationConfig, processor::Qwen3AsrProcessor},
+    }, tokenizer::TokenizerModel, utils::{
+        build_completion_chunk_response, build_completion_response, find_type_files, get_device,
+        get_dtype, get_logit_processor,
+    }
+};
+
+pub struct Qwen3AsrGenerateModel<'a> {
+    chat_template: ChatTemplate<'a>,
+    // tokenizer: TokenizerModel,
+    processor: Qwen3AsrProcessor,
+    // fun_asr_nano: FunAsrNanoModel,
+    device: Device,
+    // dtype: DType,
+    eos_token_id1: u32,
+    eos_token_id2: u32,
+    generation_config: Qwen3ASRGenerationConfig,
+    model_name: String,
+}
+
+impl<'a> Qwen3AsrGenerateModel<'a> {
+    pub fn init(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self> {
+        let chat_template = ChatTemplate::init(path)?;
+        let generation_config_path = path.to_string() + "/generation_config.json";
+        let generation_config: Qwen3ASRGenerationConfig =
+            serde_json::from_slice(&std::fs::read(generation_config_path)?)?;
+        let device = get_device(device);
+        let processor = Qwen3AsrProcessor::new(&device)?;
+
+        
+        Ok(Self {
+            chat_template,
+            // tokenizer,
+            processor,
+            // fun_asr_nano,
+            device,
+            // dtype,
+            eos_token_id1: generation_config.eos_token_id[0] as u32,
+            eos_token_id2: generation_config.eos_token_id[1] as u32,
+            generation_config,
+            model_name: "qwen3-asr".to_string(),
+        })
+    }
+
+    pub fn generate(&mut self, mes: ChatCompletionParameters) -> Result<()> {
+        let temperature = match mes.temperature {
+            None => self.generation_config.temperature,
+            Some(tem) => tem,
+        };
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let mut logit_processor =
+            get_logit_processor(Some(temperature), mes.top_p, None, seed);
+        let render_text = self.chat_template.apply_chat_template(&mes)?;
+        let audio_data =
+            self.processor.process_info(&mes, &render_text)?;
+        // for audio in audio_data {
+        //     let text = 
+        // }
+        
+        Ok(())
+    }
+
+}
+
+// impl<'a> GenerateModel for Qwen3AsrGenerateModel<'a> {
+//     fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
+//         let temperature = match mes.temperature {
+//             None => self.generation_config.temperature,
+//             Some(tem) => tem,
+//         };
+//         let seed = match mes.seed {
+//             None => 34562u64,
+//             Some(s) => s as u64,
+//         };
+//         let mut logit_processor =
+//             get_logit_processor(Some(temperature), mes.top_p, None, seed);
+//         let audio_data =
+//             self.processor.process_info(&mes)?;
+//         for audio in audio_data {
+//             let text = 
+//         }
+//         let mut speech = Some(speech.to_dtype(self.dtype)?);
+//         let mut fbank_mask = Some(&fbank_mask);
+//         let mut seq_len = input_ids.dim(1)?;
+//         let mut seqlen_offset = 0;
+//         let mut generate = Vec::new();
+//         let sample_len = mes.max_tokens.unwrap_or(1024);
+//         for _ in 0..sample_len {
+//             let logits = self.fun_asr_nano.forward(
+//                 &input_ids,
+//                 speech.as_ref(),
+//                 fbank_mask,
+//                 seqlen_offset,
+//             )?;
+//             let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
+//             let next_token = logit_processor.sample(&logits)?;
+//             generate.push(next_token);
+//             if next_token == self.eos_token_id1 || next_token == self.eos_token_id2 {
+//                 break;
+//             }
+//             seqlen_offset += seq_len;
+//             seq_len = 1;
+//             input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+//             speech = None;
+//             fbank_mask = None;
+//         }
+//         let num_token = generate.len() as u32;
+//         let res = self.tokenizer.token_decode(generate)?;
+//         self.fun_asr_nano.clear_kv_cache();
+//         let response = build_completion_response(res, &self.model_name, Some(num_token));
+//         Ok(response)
+//     }
+
+//     fn generate_stream(
+//         &mut self,
+//         mes: ChatCompletionParameters,
+//     ) -> Result<
+//         Box<
+//             dyn Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>
+//                 + Send
+//                 + Unpin
+//                 + '_,
+//         >,
+//     > {
+//         let temperature = match mes.temperature {
+//             None => self.generation_config.temperature,
+//             Some(tem) => tem,
+//         };
+//         let top_p = match mes.top_p {
+//             None => self.generation_config.top_p,
+//             Some(top_p) => top_p,
+//         };
+//         let top_k = self.generation_config.top_k;
+//         let seed = match mes.seed {
+//             None => 34562u64,
+//             Some(s) => s as u64,
+//         };
+//         let mut logit_processor =
+//             get_logit_processor(Some(temperature), Some(top_p), Some(top_k), seed);
+//         let (speech, fbank_mask, input_ids) = self.processor.process_info(&mes, &self.tokenizer)?;
+//         let mut seq_len = input_ids.dim(1)?;
+//         let mut seqlen_offset = 0;
+//         let sample_len = mes.max_tokens.unwrap_or(1024);
+//         let stream = stream! {
+//             let mut error_tokens = Vec::new();
+//             let mut speech = Some(speech.to_dtype(self.dtype)?);
+//             let mut fbank_mask = Some(&fbank_mask);
+//             let mut input_ids = input_ids;
+//             for _ in 0..sample_len {
+//                 let logits = self.fun_asr_nano.forward(
+//                     &input_ids,
+//                     speech.as_ref(),
+//                     fbank_mask,
+//                     seqlen_offset,
+//                 )?;
+//                 let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
+//                 let next_token = logit_processor.sample(&logits)?;
+//                 let mut decode_ids = Vec::new();
+//                 if !error_tokens.is_empty() {
+//                     decode_ids.extend_from_slice(&error_tokens);
+//                 }
+//                 decode_ids.push(next_token);
+//                 let decoded_token = self.tokenizer.token_decode(decode_ids).map_err(|e| anyhow!(format!("stream decode error{e}")))?;
+//                 if decoded_token.contains("�") {
+//                     error_tokens.push(next_token);
+//                     if error_tokens.len() > 3 {
+//                         error_tokens.clear();
+//                     }
+//                     seqlen_offset += seq_len;
+//                     seq_len = 1;
+//                     input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+//                     speech = None;
+//                     fbank_mask = None;
+//                     continue;
+//                 }
+//                 error_tokens.clear();
+//                 let chunk = build_completion_chunk_response(decoded_token, &self.model_name, None, None);
+//                 yield Ok(chunk);
+//                 if next_token == self.eos_token_id1 || next_token == self.eos_token_id2 {
+//                     break;
+//                 }
+//                 seqlen_offset += seq_len;
+//                 seq_len = 1;
+//                 input_ids = Tensor::from_vec(vec![next_token], (1, 1), &self.device)?;
+//                 speech = None;
+//                 fbank_mask = None;
+//             }
+//             self.fun_asr_nano.clear_kv_cache();
+//         };
+//         Ok(Box::new(Box::pin(stream)))
+//     }
+// }

src/models/qwen3_asr/mod.rs

@@ -0,0 +1,4 @@
+pub mod config;
+pub mod generate;
+pub mod model;
+pub mod processor;

src/models/qwen3_asr/model.rs

@@ -0,0 +1,646 @@
+use anyhow::Result;
+use candle_core::{D, IndexOp, Tensor};
+use candle_nn::{Conv1d, LayerNorm, Linear, Module, VarBuilder, linear, ops::softmax_last_dim};
+
+use crate::{
+    models::{
+        common::{
+            NaiveAttention, TwoLinearMLP, eager_attention_forward, get_conv1d, get_layer_norm,
+        },
+        fun_asr_nano::config::FunASRNanoConfig,
+        qwen3::{config::Qwen3Config, model::Qwen3Model},
+    },
+    position_embed::sinusoidal_pe::SinusoidalPositionEncoderCat,
+    utils::tensor_utils::{get_equal_mask, mask_filled, masked_scatter_dim0},
+};
+
+pub struct MultiHeadedAttentionSANM {
+    head_dim: usize,
+    n_head: usize,
+    linear_out: Linear,
+    linear_q_k_v: Linear,
+    fsmn_block: Conv1d,
+    left_padding: usize,
+    right_padding: usize,
+    scaling: f64,
+}
+
+impl MultiHeadedAttentionSANM {
+    pub fn new(
+        vb: VarBuilder,
+        n_head: usize,
+        in_dim: usize,
+        hidden_dim: usize,
+        kernel_size: usize,
+        sanm_shfit: usize,
+    ) -> Result<Self> {
+        let head_dim = hidden_dim / n_head;
+        let linear_out = linear(hidden_dim, hidden_dim, vb.pp("linear_out"))?;
+        let linear_q_k_v = linear(in_dim, hidden_dim * 3, vb.pp("linear_q_k_v"))?;
+        let fsmn_block = get_conv1d(
+            vb.pp("fsmn_block"),
+            hidden_dim,
+            hidden_dim,
+            kernel_size,
+            0,
+            1,
+            1,
+            hidden_dim,
+            false,
+        )?;
+        let mut left_padding = (kernel_size - 1) / 2;
+        if sanm_shfit > 0 {
+            left_padding += sanm_shfit;
+        }
+        let right_padding = kernel_size - 1 - left_padding;
+        let scaling = (head_dim as f64).powf(-0.5);
+        Ok(Self {
+            head_dim,
+            n_head,
+            linear_out,
+            linear_q_k_v,
+            fsmn_block,
+            left_padding,
+            right_padding,
+            scaling,
+        })
+    }
+
+    pub fn forward_fsmn(
+        &self,
+        inputs: &Tensor,
+        mask: Option<&Tensor>,
+        mask_shfit_chunk: Option<&Tensor>,
+    ) -> Result<Tensor> {
+        let mut inputs = inputs.clone();
+        let mask = if let Some(mask) = mask {
+            let mut mask = mask.unsqueeze(D::Minus1)?.unsqueeze(0)?;
+            if let Some(mask_shfit_chunk) = mask_shfit_chunk {
+                mask = mask.broadcast_mul(mask_shfit_chunk)?;
+            }
+            inputs = inputs.broadcast_mul(&mask)?;
+            Some(mask)
+        } else {
+            None
+        };
+        let xs = inputs.transpose(1, 2)?;
+        let xs = xs.pad_with_zeros(D::Minus1, self.left_padding, self.right_padding)?;
+        let xs = self.fsmn_block.forward(&xs)?;
+        let xs = xs.transpose(1, 2)?;
+        let mut xs = xs.add(&inputs)?;
+        if let Some(mask) = mask {
+            xs = xs.broadcast_mul(&mask)?;
+        }
+        Ok(xs)
+    }
+    pub fn forward_qkv(&self, xs: &Tensor) -> Result<(Tensor, Tensor, Tensor, Tensor)> {
+        let (b, t, _) = xs.dims3()?;
+        let q_k_v = self
+            .linear_q_k_v
+            .forward(xs)?
+            .reshape((b, t, 3, self.n_head, ()))?
+            .permute((2, 0, 3, 1, 4))?
+            .contiguous()?;
+        let q_h = q_k_v.i(0)?.contiguous()?;
+        let k_h = q_k_v.i(1)?.contiguous()?;
+        let v_h = q_k_v.i(2)?.contiguous()?;
+        let v = v_h.transpose(1, 2)?.reshape((b, t, ()))?;
+        Ok((q_h, k_h, v_h, v))
+    }
+
+    pub fn forward_attention(
+        &self,
+        values: &Tensor,
+        scores: &Tensor,
+        mask: Option<&Tensor>,
+        mask_att_chunk_encoder: Option<&Tensor>,
+    ) -> Result<Tensor> {
+        let bs = scores.dim(0)?;
+        let attn = if let Some(mask) = mask {
+            let mask = if let Some(mask_att_chunk_encoder) = mask_att_chunk_encoder {
+                mask.mul(mask_att_chunk_encoder)?
+            } else {
+                mask.clone()
+            };
+            // mask: rank = 2
+            let mask = get_equal_mask(&mask, 0)?;
+            let scores = mask_filled(scores, &mask, f32::NEG_INFINITY)?;
+            let attn = softmax_last_dim(&scores)?;
+            mask_filled(&attn, &mask, 0.0)?
+        } else {
+            softmax_last_dim(scores)?
+        };
+        let xs = attn.matmul(values)?;
+        let xs =
+            xs.transpose(1, 2)?
+                .contiguous()?
+                .reshape((bs, (), self.n_head * self.head_dim))?;
+        let xs = self.linear_out.forward(&xs)?;
+        Ok(xs)
+    }
+
+    pub fn forward_simple(&self, xs: &Tensor) -> Result<Tensor> {
+        let (b, t, _) = xs.dims3()?;
+        let q_k_v = self.linear_q_k_v.forward(xs)?;
+        let dim = self.head_dim * self.n_head;
+        let q_h = q_k_v
+            .narrow(D::Minus1, 0, dim)?
+            .reshape((b, t, self.n_head, ()))?
+            .permute((0, 2, 1, 3))?;
+        let k_h = q_k_v
+            .narrow(D::Minus1, dim, dim)?
+            .reshape((b, t, self.n_head, ()))?
+            .permute((0, 2, 1, 3))?;
+        let v = q_k_v.narrow(D::Minus1, dim * 2, dim)?;
+        let v_h = v.reshape((b, t, self.n_head, ()))?.permute((0, 2, 1, 3))?;
+        let fsmn_memory = v.transpose(1, 2)?;
+        let fsmn_memory = fsmn_memory
+            .pad_with_zeros(D::Minus1, self.left_padding, self.right_padding)?
+            .contiguous()?;
+        let fsmn_memory = self.fsmn_block.forward(&fsmn_memory)?;
+        // let fsmn_memory = conv1d_group_parallel(&fsmn_memory, &self.fsmn_block)?;
+
+        let fsmn_memory = fsmn_memory.transpose(1, 2)?;
+        let fsmn_memory = fsmn_memory.add(&v)?;
+        let att_outs = eager_attention_forward(&q_h, &k_h, &v_h, None, None, self.scaling)?;
+        let att_outs = att_outs.reshape((b, t, ()))?;
+        let att_outs = self.linear_out.forward(&att_outs)?;
+        let att_outs = att_outs.add(&fsmn_memory)?;
+        Ok(att_outs)
+    }
+
+    pub fn forward(
+        &self,
+        xs: &Tensor,
+        mask: Option<&Tensor>,
+        mask_shfit_chunk: Option<&Tensor>,
+        mask_att_chunk_encoder: Option<&Tensor>,
+    ) -> Result<Tensor> {
+        let (q_h, k_h, v_h, v) = self.forward_qkv(xs)?;
+        let fsmn_memory = self.forward_fsmn(&v, mask, mask_shfit_chunk)?;
+        let q_h = q_h.affine(self.scaling, 0.0)?;
+        let scores = q_h.matmul(&k_h.transpose(D::Minus2, D::Minus1)?)?;
+        let attn_outs = self.forward_attention(&v_h, &scores, mask, mask_att_chunk_encoder)?;
+        let att_outs = attn_outs.add(&fsmn_memory)?;
+        Ok(att_outs)
+    }
+}
+
+pub struct EncoderLayerSANM {
+    self_attn: MultiHeadedAttentionSANM,
+    feed_forward: TwoLinearMLP,
+    norm1: LayerNorm,
+    norm2: LayerNorm,
+    concat_linear: Option<Linear>,
+    normalize_before: bool,
+    in_dim: usize,
+    hidden_dim: usize,
+}
+
+impl EncoderLayerSANM {
+    pub fn new(
+        vb: VarBuilder,
+        in_dim: usize,
+        hidden_dim: usize,
+        n_head: usize,
+        kernel_size: usize,
+        sanm_shfit: usize,
+        hidden_units: usize,
+        normalize_before: bool,
+        concat_after: bool,
+    ) -> Result<Self> {
+        let self_attn = MultiHeadedAttentionSANM::new(
+            vb.pp("self_attn"),
+            n_head,
+            in_dim,
+            hidden_dim,
+            kernel_size,
+            sanm_shfit,
+        )?;
+        let feed_forward = TwoLinearMLP::new(
+            vb.pp("feed_forward"),
+            hidden_dim,
+            hidden_units,
+            hidden_dim,
+            candle_nn::Activation::Relu,
+            true,
+            "w_1",
+            "w_2",
+        )?;
+        let norm1 = get_layer_norm(vb.pp("norm1"), 1e-5, in_dim)?;
+        let norm2 = get_layer_norm(vb.pp("norm2"), 1e-5, hidden_dim)?;
+        let concat_linear = if concat_after {
+            let lin = linear(hidden_dim * 2, hidden_dim, vb.pp("concat_linear"))?;
+            Some(lin)
+        } else {
+            None
+        };
+        Ok(Self {
+            self_attn,
+            feed_forward,
+            norm1,
+            norm2,
+            concat_linear,
+            normalize_before,
+            in_dim,
+            hidden_dim,
+        })
+    }
+
+    pub fn forward(
+        &self,
+        xs: &Tensor,
+        mask: Option<&Tensor>,
+        mask_shfit_chunk: Option<&Tensor>,
+        mask_att_chunk_encoder: Option<&Tensor>,
+    ) -> Result<Tensor> {
+        let stoch_layer_coeff = 1.0f64;
+        let residual = xs.clone();
+        let mut xs = if self.normalize_before {
+            self.norm1.forward(xs)?
+        } else {
+            xs.clone()
+        };
+        if self.concat_linear.is_some() {
+            let attn =
+                self.self_attn
+                    .forward(&xs, mask, mask_shfit_chunk, mask_att_chunk_encoder)?;
+            let x_concat = Tensor::cat(&[&xs, &attn], D::Minus1)?;
+            if self.in_dim == self.hidden_dim {
+                let x_concat = self
+                    .concat_linear
+                    .as_ref()
+                    .unwrap()
+                    .forward(&x_concat)?
+                    .affine(stoch_layer_coeff, 0.0)?;
+                xs = residual.add(&x_concat)?;
+            } else {
+                xs = self
+                    .concat_linear
+                    .as_ref()
+                    .unwrap()
+                    .forward(&x_concat)?
+                    .affine(stoch_layer_coeff, 0.0)?;
+            }
+        } else if self.in_dim == self.hidden_dim {
+            let attn = self
+                .self_attn
+                .forward(&xs, mask, mask_shfit_chunk, mask_att_chunk_encoder)?
+                .affine(stoch_layer_coeff, 0.0)?;
+            xs = residual.add(&attn)?;
+        } else {
+            xs = self
+                .self_attn
+                .forward(&xs, mask, mask_shfit_chunk, mask_att_chunk_encoder)?
+                .affine(stoch_layer_coeff, 0.0)?;
+        }
+
+        if !self.normalize_before {
+            xs = self.norm1.forward(&xs)?;
+        }
+        let residual = xs.clone();
+        if self.normalize_before {
+            xs = self.norm2.forward(&xs)?;
+        }
+        xs = self
+            .feed_forward
+            .forward(&xs)?
+            .affine(stoch_layer_coeff, 0.0)?;
+        xs = residual.add(&xs)?;
+        if !self.normalize_before {
+            xs = self.norm2.forward(&xs)?;
+        }
+        Ok(xs)
+    }
+
+    pub fn forward_simple(&self, xs: &Tensor) -> Result<Tensor> {
+        let residual = xs.clone();
+        let mut xs = self.norm1.forward(xs)?;
+        if self.in_dim == self.hidden_dim {
+            let attn = self.self_attn.forward_simple(&xs)?;
+            xs = residual.add(&attn)?;
+        } else {
+            xs = self.self_attn.forward_simple(&xs)?;
+        }
+
+        let residual = xs.clone();
+        let xs = self.norm2.forward(&xs)?;
+
+        let xs = self.feed_forward.forward(&xs)?;
+        let xs = residual.add(&xs)?;
+        Ok(xs)
+    }
+}
+
+pub struct SenseVoiceEncoderSmall {
+    embed: SinusoidalPositionEncoderCat,
+    encoders0: EncoderLayerSANM,
+    encoders: Vec<EncoderLayerSANM>,
+    tp_encoders: Vec<EncoderLayerSANM>,
+    after_norm: LayerNorm,
+    tp_norm: LayerNorm,
+    scaling: f64,
+}
+
+impl SenseVoiceEncoderSmall {
+    pub fn new(
+        vb: VarBuilder,
+        input_size: usize,
+        output_size: usize,
+        attention_heads: usize,
+        linear_units: usize,
+        num_blocks: usize,
+        tp_blocks: usize,
+        normalize_before: bool,
+        kernel_size: usize,
+        sanm_shfit: usize,
+    ) -> Result<Self> {
+        let embed = SinusoidalPositionEncoderCat::new(Some(input_size), true, vb.device())?;
+
+        let encoders0 = EncoderLayerSANM::new(
+            vb.pp("encoders0.0"),
+            input_size,
+            output_size,
+            attention_heads,
+            kernel_size,
+            sanm_shfit,
+            linear_units,
+            normalize_before,
+            false,
+        )?;
+        let mut encoders = vec![];
+        let vb_encoders = vb.pp("encoders");
+        for i in 0..(num_blocks - 1) {
+            let encoder_i = EncoderLayerSANM::new(
+                vb_encoders.pp(i),
+                output_size,
+                output_size,
+                attention_heads,
+                kernel_size,
+                sanm_shfit,
+                linear_units,
+                normalize_before,
+                false,
+            )?;
+            encoders.push(encoder_i);
+        }
+        let vb_tp_encoders = vb.pp("tp_encoders");
+        let mut tp_encoders = vec![];
+        for i in 0..tp_blocks {
+            let tp_blocks_i = EncoderLayerSANM::new(
+                vb_tp_encoders.pp(i),
+                output_size,
+                output_size,
+                attention_heads,
+                kernel_size,
+                sanm_shfit,
+                linear_units,
+                normalize_before,
+                false,
+            )?;
+            tp_encoders.push(tp_blocks_i);
+        }
+        let after_norm = get_layer_norm(vb.pp("after_norm"), 1e-5, output_size)?;
+        let tp_norm = get_layer_norm(vb.pp("tp_norm"), 1e-5, output_size)?;
+        let scaling = (output_size as f64).powf(0.5);
+        Ok(Self {
+            embed,
+            encoders0,
+            encoders,
+            tp_encoders,
+            after_norm,
+            tp_norm,
+            scaling,
+        })
+    }
+    pub fn forward(&self, xs: &Tensor) -> Result<Tensor> {
+        let xs = xs.affine(self.scaling, 0.0)?;
+        let xs = self.embed.forward(&xs, 0)?;
+        let mut xs = self.encoders0.forward_simple(&xs)?;
+        for encoder_layer in &self.encoders {
+            xs = encoder_layer.forward_simple(&xs)?;
+        }
+        xs = self.after_norm.forward(&xs)?;
+        for tp_layer in &self.tp_encoders {
+            xs = tp_layer.forward_simple(&xs)?;
+        }
+        xs = self.tp_norm.forward(&xs)?;
+        Ok(xs)
+    }
+}
+
+pub struct AdaptorEncoderLayer {
+    self_attn: NaiveAttention,
+    feed_forward: TwoLinearMLP,
+    norm1: LayerNorm,
+    norm2: LayerNorm,
+    concat_linear: Option<Linear>,
+    normalize_before: bool,
+}
+
+impl AdaptorEncoderLayer {
+    pub fn new(
+        vb: VarBuilder,
+        llm_dim: usize,
+        n_head: usize,
+        normalize_before: bool,
+        concat_after: bool,
+    ) -> Result<Self> {
+        let self_attn = NaiveAttention::new(
+            vb.pp("self_attn"),
+            llm_dim,
+            n_head,
+            n_head,
+            None,
+            true,
+            Some("linear_q"),
+            Some("linear_k"),
+            Some("linear_v"),
+            Some("linear_out"),
+        )?;
+        let feed_forward = TwoLinearMLP::new(
+            vb.pp("feed_forward"),
+            llm_dim,
+            llm_dim / 4,
+            llm_dim,
+            candle_nn::Activation::Relu,
+            true,
+            "w_1",
+            "w_2",
+        )?;
+        let norm1 = get_layer_norm(vb.pp("norm1"), 1e-5, llm_dim)?;
+        let norm2 = get_layer_norm(vb.pp("norm2"), 1e-5, llm_dim)?;
+        let concat_linear = if concat_after {
+            let lin = linear(llm_dim * 2, llm_dim, vb.pp("concat_linear"))?;
+            Some(lin)
+        } else {
+            None
+        };
+        Ok(Self {
+            self_attn,
+            feed_forward,
+            norm1,
+            norm2,
+            concat_linear,
+            normalize_before,
+        })
+    }
+
+    pub fn forward(&self, xs: &Tensor, mask: Option<&Tensor>) -> Result<Tensor> {
+        let stoch_layer_coeff = 1.0f64;
+        let residual = xs.clone();
+        let mut xs = if self.normalize_before {
+            self.norm1.forward(xs)?
+        } else {
+            xs.clone()
+        };
+        if self.concat_linear.is_some() {
+            let attn = self.self_attn.forward(&xs, None, None, mask, false)?;
+            let x_concat = Tensor::cat(&[&xs, &attn], D::Minus1)?;
+            let x_concat = self
+                .concat_linear
+                .as_ref()
+                .unwrap()
+                .forward(&x_concat)?
+                .affine(stoch_layer_coeff, 0.0)?;
+            xs = residual.add(&x_concat)?;
+        } else {
+            let attn = self
+                .self_attn
+                .forward(&xs, None, None, mask, false)?
+                .affine(stoch_layer_coeff, 0.0)?;
+            xs = residual.add(&attn)?;
+        }
+        if !self.normalize_before {
+            xs = self.norm1.forward(&xs)?;
+        }
+        let residual = xs.clone();
+        if self.normalize_before {
+            xs = self.norm2.forward(&xs)?;
+        }
+        xs = self
+            .feed_forward
+            .forward(&xs)?
+            .affine(stoch_layer_coeff, 0.0)?;
+        xs = residual.add(&xs)?;
+        if !self.normalize_before {
+            xs = self.norm2.forward(&xs)?;
+        }
+        Ok(xs)
+    }
+}
+
+pub struct AudioAdaptor {
+    k: usize,
+    linear1: Linear,
+    linear2: Linear,
+    blocks: Vec<AdaptorEncoderLayer>,
+}
+
+impl AudioAdaptor {
+    pub fn new(
+        vb: VarBuilder,
+        downsample_rate: usize,
+        encoder_dim: usize,
+        llm_dim: usize,
+        ffn_dim: usize,
+        n_layer: usize,
+        attention_heads: usize,
+    ) -> Result<Self> {
+        let linear1 = linear(encoder_dim * downsample_rate, ffn_dim, vb.pp("linear1"))?;
+        let linear2 = linear(ffn_dim, llm_dim, vb.pp("linear2"))?;
+        let mut blocks = vec![];
+        let vb_blocks = vb.pp("blocks");
+        for i in 0..n_layer {
+            let layer =
+                AdaptorEncoderLayer::new(vb_blocks.pp(i), llm_dim, attention_heads, true, false)?;
+            blocks.push(layer);
+        }
+        Ok(Self {
+            k: downsample_rate,
+            linear1,
+            linear2,
+            blocks,
+        })
+    }
+    pub fn forward(&self, xs: &Tensor) -> Result<Tensor> {
+        let (bs, seq_len, dim) = xs.dims3()?;
+        let chunk_num = (seq_len - 1) / self.k + 1;
+        let pad_num = chunk_num * self.k - seq_len;
+        let xs = xs.pad_with_zeros(1, 0, pad_num)?;
+        let xs = xs.contiguous()?.reshape((bs, chunk_num, dim * self.k))?;
+        let xs = self.linear1.forward(&xs)?.relu()?;
+        let mut xs = self.linear2.forward(&xs)?;
+        for block in &self.blocks {
+            xs = block.forward(&xs, None)?;
+        }
+        Ok(xs)
+    }
+}
+
+pub struct FunAsrNanoModel {
+    audio_encoder: SenseVoiceEncoderSmall,
+    audio_adaptor: AudioAdaptor,
+    llm: Qwen3Model,
+}
+impl FunAsrNanoModel {
+    pub fn new(vb: VarBuilder, config: &FunASRNanoConfig, llm_cfg: &Qwen3Config) -> Result<Self> {
+        let input_size = config.frontend_conf.lfr_m * config.frontend_conf.n_mels;
+        let audio_encoder = SenseVoiceEncoderSmall::new(
+            vb.pp("audio_encoder"),
+            input_size,
+            config.audio_encoder_conf.output_size,
+            config.audio_encoder_conf.attention_heads,
+            config.audio_encoder_conf.linear_units,
+            config.audio_encoder_conf.num_blocks,
+            config.audio_encoder_conf.tp_blocks,
+            config.audio_encoder_conf.normalize_before,
+            config.audio_encoder_conf.kernel_size,
+            config.audio_encoder_conf.sanm_shfit,
+        )?;
+        let audio_adaptor = AudioAdaptor::new(
+            vb.pp("audio_adaptor"),
+            config.audio_adaptor_conf.downsample_rate,
+            config.audio_adaptor_conf.encoder_dim,
+            config.audio_adaptor_conf.llm_dim,
+            config.audio_adaptor_conf.ffn_dim,
+            config.audio_adaptor_conf.n_layer,
+            8,
+        )?;
+        let llm = Qwen3Model::new(llm_cfg, vb.pp("llm"))?;
+        Ok(Self {
+            audio_encoder,
+            audio_adaptor,
+            llm,
+        })
+    }
+
+    pub fn forward(
+        &mut self,
+        input_ids: &Tensor,
+        speech: Option<&Tensor>,
+        fbank_mask: Option<&Tensor>,
+        seqlen_offset: usize,
+    ) -> Result<Tensor> {
+        let mut inputs_embeds = self.llm.embedding_token_id(input_ids)?;
+        if let Some(speech) = speech
+            && let Some(fbank_mask) = fbank_mask
+        {
+            let speech = self.audio_encoder.forward(speech)?;
+            let encoder_out = self.audio_adaptor.forward(&speech)?;
+            let speech_token_len = fbank_mask.sum_all()?.to_scalar::<u32>()?;
+            let audio_embed = encoder_out
+                .squeeze(0)?
+                .narrow(0, 0, speech_token_len as usize)?;
+            inputs_embeds = masked_scatter_dim0(&inputs_embeds, &audio_embed, fbank_mask)?;
+        }
+        let logits = self
+            .llm
+            .forward(None, Some(&inputs_embeds), seqlen_offset)?;
+        Ok(logits)
+    }
+
+    pub fn clear_kv_cache(&mut self) {
+        self.llm.clear_kv_cache();
+    }
+}

src/models/qwen3_asr/processor.rs

@@ -0,0 +1,127 @@
+use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
+use anyhow::Result;
+use candle_core::{Device, Tensor};
+use serde_json::{Value, json};
+
+use crate::utils::{
+    audio_utils::{extract_audios, split_audio_into_chunks},
+    capitalize_first_letter, extract_user_text_vec,
+    tensor_utils::float_range_normalize,
+};
+
+pub struct Qwen3AsrProcessor {
+    device: Device,
+    sample_rate: usize,
+    support_language: Vec<String>,
+    max_asr_input_seconds: f32,
+}
+
+impl Qwen3AsrProcessor {
+    pub fn new(device: &Device) -> Result<Self> {
+        let support_language: Vec<String> = vec![
+            "Chinese",
+            "English",
+            "Cantonese",
+            "Arabic",
+            "German",
+            "French",
+            "Spanish",
+            "Portuguese",
+            "Indonesian",
+            "Italian",
+            "Korean",
+            "Russian",
+            "Thai",
+            "Vietnamese",
+            "Japanese",
+            "Turkish",
+            "Hindi",
+            "Malay",
+            "Dutch",
+            "Swedish",
+            "Danish",
+            "Finnish",
+            "Polish",
+            "Czech",
+            "Filipino",
+            "Persian",
+            "Greek",
+            "Romanian",
+            "Hungarian",
+            "Macedonian",
+        ]
+        .iter()
+        .map(|s| s.to_string())
+        .collect();
+        Ok(Self {
+            device: device.clone(),
+            sample_rate: 16000,
+            support_language,
+            max_asr_input_seconds: 1200.0,
+        })
+    }
+
+    pub fn process_audio(&self, mes: &ChatCompletionParameters) -> Result<Vec<Tensor>> {
+        let audio_tensors = extract_audios(mes, &self.device, Some(self.sample_rate))?;
+        audio_tensors
+            .iter()
+            .map(|audio| float_range_normalize(&audio))
+            .collect()
+    }
+
+    pub fn validate_language(&self, lang: &String) -> bool {
+        self.support_language.contains(lang)
+    }
+
+    pub fn process_info(&self, mes: &ChatCompletionParameters, render: &str) -> Result<()> {
+        let audio_count = render
+            .matches("<|audio_start|><|audio_pad|><|audio_end|>")
+            .count();
+        let mut render = if audio_count > 1 {
+            render.replace(
+                &"<|audio_start|><|audio_pad|><|audio_end|>".repeat(audio_count),
+                "<|audio_start|><|audio_pad|><|audio_end|>",
+            )
+        } else {
+            render.to_string()
+        };
+        if let Some(map) = &mes.metadata
+            && map.contains_key("language")
+        {
+            let lang = map.get("language").unwrap();
+            let lang = capitalize_first_letter(lang);
+            if self.validate_language(&lang) {
+                render = format!("{}language {}'<asr_text>'", render, lang);
+            }
+        }
+        let audio_tensors = self.process_audio(mes)?;
+        let audio_len = audio_tensors.len();
+        if audio_len != audio_count {
+            return Err(anyhow::anyhow!("audio_pad num != audio num"));
+        }
+        let mut split_wavs = vec![];
+        for wav in audio_tensors.iter() {
+            let wavs = split_audio_into_chunks(wav, self.sample_rate, self.max_asr_input_seconds)?;
+            split_wavs.extend_from_slice(&wavs);
+        }
+        
+
+        // let mut audio_datas = vec![];
+        // for (i, wav) in audio_tensors.iter().enumerate() {
+        //     let wavs = split_audio_into_chunks(wav, self.sample_rate, self.max_asr_input_seconds)?;
+        //     for i_w in wavs {
+        //         let audio_data = AudioData {
+        //             wav: i_w,
+        //             language: langs[i].clone(),
+        //         };
+        //         audio_datas.push(audio_data);
+        //     }
+        // }
+        Ok(())
+    }
+}
+
+pub struct AudioData {
+    pub wav: Tensor,
+    pub language: Option<String>,
+}