refactor: reorganize imports and improve code formatting across multiple modules
This commit is contained in:
@@ -5,14 +5,13 @@ use std::time::Instant;
|
|||||||
use anyhow::{Ok, Result};
|
use anyhow::{Ok, Result};
|
||||||
|
|
||||||
use crate::exec::ExecModel;
|
use crate::exec::ExecModel;
|
||||||
|
use crate::models::GenerateModel;
|
||||||
use crate::models::qwen3_asr::generate::Qwen3AsrGenerateModel;
|
use crate::models::qwen3_asr::generate::Qwen3AsrGenerateModel;
|
||||||
use crate::models::{GenerateModel};
|
|
||||||
|
|
||||||
pub struct Qwen3ASRExec;
|
pub struct Qwen3ASRExec;
|
||||||
|
|
||||||
impl ExecModel for Qwen3ASRExec {
|
impl ExecModel for Qwen3ASRExec {
|
||||||
fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
|
fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
|
||||||
|
|
||||||
let i_start = Instant::now();
|
let i_start = Instant::now();
|
||||||
let mut model = Qwen3AsrGenerateModel::init(weight_path, None, None)?;
|
let mut model = Qwen3AsrGenerateModel::init(weight_path, None, None)?;
|
||||||
let i_duration = i_start.elapsed();
|
let i_duration = i_start.elapsed();
|
||||||
|
|||||||
@@ -25,7 +25,11 @@ impl Shortcut {
|
|||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let conv_0 = get_conv2d(vb.pp("0"), in_c, out_c, ks, padding, 1, 1, 1, bias)?;
|
let conv_0 = get_conv2d(vb.pp("0"), in_c, out_c, ks, padding, 1, 1, 1, bias)?;
|
||||||
let bn_1 = get_batch_norm(vb.pp("1"), 1e-5, out_c, true)?;
|
let bn_1 = get_batch_norm(vb.pp("1"), 1e-5, out_c, true)?;
|
||||||
Ok(Self { conv_0, bn_1, stride })
|
Ok(Self {
|
||||||
|
conv_0,
|
||||||
|
bn_1,
|
||||||
|
stride,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
|
pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
|
||||||
@@ -36,7 +40,7 @@ impl Shortcut {
|
|||||||
let indices = Tensor::arange(0u32, half_h as u32, x.device())?.affine(2.0, 0.0)?;
|
let indices = Tensor::arange(0u32, half_h as u32, x.device())?.affine(2.0, 0.0)?;
|
||||||
x = x.index_select(&indices, 2)?;
|
x = x.index_select(&indices, 2)?;
|
||||||
}
|
}
|
||||||
x = self.bn_1.forward_t(&x, false)?;
|
x = self.bn_1.forward_t(&x, false)?;
|
||||||
Ok(x)
|
Ok(x)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -106,7 +110,7 @@ impl BasicResBlock {
|
|||||||
} else {
|
} else {
|
||||||
xs = xs.add(&residual)?;
|
xs = xs.add(&residual)?;
|
||||||
}
|
}
|
||||||
xs = xs.relu()?;
|
xs = xs.relu()?;
|
||||||
Ok(xs)
|
Ok(xs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,4 +18,4 @@ pub struct FeatureExtractor {
|
|||||||
|
|
||||||
fn default_sampling_rate() -> usize {
|
fn default_sampling_rate() -> usize {
|
||||||
16000
|
16000
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
pub mod seamless_m4t_feature_extractor;
|
pub mod config;
|
||||||
pub mod feature_extraction_whisper;
|
pub mod feature_extraction_whisper;
|
||||||
pub mod config;
|
pub mod seamless_m4t_feature_extractor;
|
||||||
|
|||||||
@@ -11,8 +11,6 @@ pub struct GlmAsrNanoProcessorConfig {
|
|||||||
pub max_audio_len: usize,
|
pub max_audio_len: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||||
pub struct GlmAsrNanoConfig {
|
pub struct GlmAsrNanoConfig {
|
||||||
pub audio_config: GlmAsrAudioConfig,
|
pub audio_config: GlmAsrAudioConfig,
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
|
|
||||||
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
|
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use candle_core::{D, DType, Device, IndexOp, Tensor};
|
use candle_core::{D, DType, Device, IndexOp, Tensor};
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ impl IndexTTS2Generate {
|
|||||||
let device = get_device(device);
|
let device = get_device(device);
|
||||||
let dtype = get_dtype(dtype, "bf16");
|
let dtype = get_dtype(dtype, "bf16");
|
||||||
let processor = IndexTTS2Processor::new(path, &save_dir, &config, &device, dtype)?;
|
let processor = IndexTTS2Processor::new(path, &save_dir, &config, &device, dtype)?;
|
||||||
|
|
||||||
Ok(Self { config, processor })
|
Ok(Self { config, processor })
|
||||||
}
|
}
|
||||||
pub fn generate(&mut self, mes: ChatCompletionParameters) -> Result<()> {
|
pub fn generate(&mut self, mes: ChatCompletionParameters) -> Result<()> {
|
||||||
|
|||||||
@@ -2,4 +2,4 @@ pub mod config;
|
|||||||
pub mod generate;
|
pub mod generate;
|
||||||
pub mod model;
|
pub mod model;
|
||||||
pub mod processor;
|
pub mod processor;
|
||||||
pub mod utils;
|
pub mod utils;
|
||||||
|
|||||||
@@ -5,13 +5,16 @@ use candle_nn::VarBuilder;
|
|||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
models::{
|
models::{
|
||||||
campplus::CAMPPlus, feature_extractor::seamless_m4t_feature_extractor::SeamlessM4TFeatureExtractor, index_tts2::config::{IndexTTS2Config, PreprocessParams}, mask_gct::model::RepCodec, w2v_bert_2_0::model::W2VBert2_0Model
|
campplus::CAMPPlus,
|
||||||
|
feature_extractor::seamless_m4t_feature_extractor::SeamlessM4TFeatureExtractor,
|
||||||
|
index_tts2::config::{IndexTTS2Config, PreprocessParams},
|
||||||
|
mask_gct::model::RepCodec,
|
||||||
|
w2v_bert_2_0::model::W2VBert2_0Model,
|
||||||
},
|
},
|
||||||
utils::{
|
utils::{
|
||||||
audio_utils::{
|
audio_utils::{
|
||||||
create_hann_window, extract_audio_url, get_waveform_and_window_properties, kaldi_fbank,
|
create_hann_window, extract_audio_url, get_waveform_and_window_properties, kaldi_fbank,
|
||||||
kaldi_get_mel_banks, load_audio, mel_filter_bank, resample_simple,
|
kaldi_get_mel_banks, load_audio, mel_filter_bank, resample_simple, torch_stft,
|
||||||
torch_stft,
|
|
||||||
},
|
},
|
||||||
get_vb_model_path,
|
get_vb_model_path,
|
||||||
tensor_utils::pad_reflect_last_dim,
|
tensor_utils::pad_reflect_last_dim,
|
||||||
|
|||||||
@@ -7,12 +7,12 @@ pub async fn download_index_tts2_need_model(save_dir: Option<&str>) -> anyhow::R
|
|||||||
};
|
};
|
||||||
|
|
||||||
let w2v_bert2_0 = "facebook/w2v-bert-2.0";
|
let w2v_bert2_0 = "facebook/w2v-bert-2.0";
|
||||||
let mask_gct= "amphion/MaskGCT";
|
let mask_gct = "amphion/MaskGCT";
|
||||||
// let campplus= "funasr/campplus"; // huggingface
|
// let campplus= "funasr/campplus"; // huggingface
|
||||||
let campplus = "iic/speech_campplus_sv_zh-cn_16k-common"; // modelscope
|
let campplus = "iic/speech_campplus_sv_zh-cn_16k-common"; // modelscope
|
||||||
download_model(w2v_bert2_0, &save_dir, 3).await?;
|
download_model(w2v_bert2_0, &save_dir, 3).await?;
|
||||||
download_model(mask_gct, &save_dir, 3).await?;
|
download_model(mask_gct, &save_dir, 3).await?;
|
||||||
download_model(campplus, &save_dir, 3).await?;
|
download_model(campplus, &save_dir, 3).await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,4 +18,4 @@ fn default_num_quantizers() -> usize {
|
|||||||
|
|
||||||
fn default_downsample_scale() -> usize {
|
fn default_downsample_scale() -> usize {
|
||||||
1
|
1
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
|
pub mod config;
|
||||||
pub mod model;
|
pub mod model;
|
||||||
pub mod config;
|
|
||||||
@@ -116,10 +116,28 @@ impl FactorizedVectorQuantize {
|
|||||||
use_l2_normlize: bool,
|
use_l2_normlize: bool,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let (in_project, out_project) = if input_dim != codebook_dim {
|
let (in_project, out_project) = if input_dim != codebook_dim {
|
||||||
let in_project =
|
let in_project = WNConv1d::new(
|
||||||
WNConv1d::new(vb.pp("in_project"), input_dim, codebook_dim, 1, 1, 0, 1, 1, true)?;
|
vb.pp("in_project"),
|
||||||
let out_project =
|
input_dim,
|
||||||
WNConv1d::new(vb.pp("out_project"), codebook_dim, input_dim, 1, 1, 0, 1, 1, true)?;
|
codebook_dim,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
true,
|
||||||
|
)?;
|
||||||
|
let out_project = WNConv1d::new(
|
||||||
|
vb.pp("out_project"),
|
||||||
|
codebook_dim,
|
||||||
|
input_dim,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
true,
|
||||||
|
)?;
|
||||||
(Some(in_project), Some(out_project))
|
(Some(in_project), Some(out_project))
|
||||||
} else {
|
} else {
|
||||||
(None, None)
|
(None, None)
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
|
|
||||||
use aha_openai_dive::v1::resources::chat::{
|
use aha_openai_dive::v1::resources::chat::{
|
||||||
ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse,
|
ChatCompletionChunkResponse, ChatCompletionParameters, ChatCompletionResponse,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -58,4 +58,4 @@ pub struct W2VBert2_0Config {
|
|||||||
pub use_weighted_layer_sum: bool,
|
pub use_weighted_layer_sum: bool,
|
||||||
pub vocab_size: Option<usize>,
|
pub vocab_size: Option<usize>,
|
||||||
pub xvector_output_dim: usize,
|
pub xvector_output_dim: usize,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod model;
|
pub mod model;
|
||||||
|
|||||||
@@ -7,9 +7,7 @@ use candle_nn::{
|
|||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
models::{
|
models::{
|
||||||
common::{
|
common::{GLU, TwoLinearMLP, eager_attention_forward, get_conv1d, get_layer_norm},
|
||||||
GLU, TwoLinearMLP, eager_attention_forward, get_conv1d, get_layer_norm,
|
|
||||||
},
|
|
||||||
w2v_bert_2_0::config::W2VBert2_0Config,
|
w2v_bert_2_0::config::W2VBert2_0Config,
|
||||||
},
|
},
|
||||||
position_embed::rope::{RoPE, apply_rotary_pos_emb},
|
position_embed::rope::{RoPE, apply_rotary_pos_emb},
|
||||||
@@ -488,7 +486,7 @@ impl Wav2Vec2BertEncoder {
|
|||||||
for (i, layer) in (&self.layers).iter().enumerate() {
|
for (i, layer) in (&self.layers).iter().enumerate() {
|
||||||
if output_hidden_states {
|
if output_hidden_states {
|
||||||
hidden_states.push(xs.clone());
|
hidden_states.push(xs.clone());
|
||||||
}
|
}
|
||||||
if let Some(id) = layer_id
|
if let Some(id) = layer_id
|
||||||
&& id == i
|
&& id == i
|
||||||
{
|
{
|
||||||
@@ -500,7 +498,7 @@ impl Wav2Vec2BertEncoder {
|
|||||||
sin.as_ref(),
|
sin.as_ref(),
|
||||||
attention_mask.as_ref(),
|
attention_mask.as_ref(),
|
||||||
conv_attention_mask,
|
conv_attention_mask,
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
let hidden_states = if hidden_states.len() > 0 {
|
let hidden_states = if hidden_states.len() > 0 {
|
||||||
Some(hidden_states)
|
Some(hidden_states)
|
||||||
|
|||||||
@@ -350,7 +350,7 @@ impl Qwen3VLTextRotaryEmbedding {
|
|||||||
|
|
||||||
// for dim in 1..3 {
|
// for dim in 1..3 {
|
||||||
for (dim, offset) in (1..3).enumerate() {
|
for (dim, offset) in (1..3).enumerate() {
|
||||||
let dim = dim +1;
|
let dim = dim + 1;
|
||||||
let length = mrope_section[dim];
|
let length = mrope_section[dim];
|
||||||
let idx = Tensor::arange_step(offset as u32, length as u32, 3, freqs.device())?;
|
let idx = Tensor::arange_step(offset as u32, length as u32, 3, freqs.device())?;
|
||||||
let src = freqs.i(dim)?.contiguous()?; // (bs, seq_len, head_dim //2)
|
let src = freqs.i(dim)?.contiguous()?; // (bs, seq_len, head_dim //2)
|
||||||
|
|||||||
@@ -32,7 +32,9 @@ use symphonia::core::meta::MetadataOptions;
|
|||||||
use symphonia::core::probe::Hint;
|
use symphonia::core::probe::Hint;
|
||||||
|
|
||||||
use crate::utils::get_default_save_dir;
|
use crate::utils::get_default_save_dir;
|
||||||
use crate::utils::tensor_utils::{linspace, log10, pad_reflect_last_dim, pad_replicate_last_dim, split_tensor};
|
use crate::utils::tensor_utils::{
|
||||||
|
linspace, log10, pad_reflect_last_dim, pad_replicate_last_dim, split_tensor,
|
||||||
|
};
|
||||||
|
|
||||||
// 重采样方法枚举
|
// 重采样方法枚举
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ pub fn split_tensor_with_size<D: Dim>(
|
|||||||
// "input tensor dim size % splits_size must be equal to 0"
|
// "input tensor dim size % splits_size must be equal to 0"
|
||||||
// );
|
// );
|
||||||
for (i, split) in (0..dim_size).step_by(splits_size).enumerate() {
|
for (i, split) in (0..dim_size).step_by(splits_size).enumerate() {
|
||||||
let size = splits_size.min(dim_size - i*splits_size);
|
let size = splits_size.min(dim_size - i * splits_size);
|
||||||
split_res.push(t.narrow(dim, split, size)?);
|
split_res.push(t.narrow(dim, split, size)?);
|
||||||
}
|
}
|
||||||
Ok(split_res)
|
Ok(split_res)
|
||||||
|
|||||||
+7
-4
@@ -2,9 +2,9 @@
|
|||||||
|
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use aha::utils::{tensor_utils::interpolate_nearest_1d};
|
use aha::utils::tensor_utils::interpolate_nearest_1d;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use candle_core::{Tensor};
|
use candle_core::Tensor;
|
||||||
// use symphonia::core::io::MediaSourceStream;
|
// use symphonia::core::io::MediaSourceStream;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -16,8 +16,11 @@ fn messy_test() -> Result<()> {
|
|||||||
let i_start = Instant::now();
|
let i_start = Instant::now();
|
||||||
let t_inter = interpolate_nearest_1d(&t, 20)?;
|
let t_inter = interpolate_nearest_1d(&t, 20)?;
|
||||||
let i_duration = i_start.elapsed();
|
let i_duration = i_start.elapsed();
|
||||||
println!("Time elapsed in interpolate_nearest_1d is: {:?}", i_duration);
|
println!(
|
||||||
println!("t_inter: {}", t_inter);
|
"Time elapsed in interpolate_nearest_1d is: {:?}",
|
||||||
|
i_duration
|
||||||
|
);
|
||||||
|
println!("t_inter: {}", t_inter);
|
||||||
// let url = "https://sis-sample-audio.obs.cn-north-1.myhuaweicloud.com/16k16bit.mp3";
|
// let url = "https://sis-sample-audio.obs.cn-north-1.myhuaweicloud.com/16k16bit.mp3";
|
||||||
// let client = reqwest::blocking::Client::new();
|
// let client = reqwest::blocking::Client::new();
|
||||||
// let response = client.get(url).send()?;
|
// let response = client.get(url).send()?;
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use anyhow::Result;
|
|
||||||
|
|
||||||
use aha::models::index_tts2::{generate::IndexTTS2Generate, utils::download_index_tts2_need_model};
|
use aha::models::index_tts2::{generate::IndexTTS2Generate, utils::download_index_tts2_need_model};
|
||||||
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
|
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
|
||||||
|
use anyhow::Result;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn index_tts2_generate() -> Result<()> {
|
async fn index_tts2_generate() -> Result<()> {
|
||||||
@@ -45,4 +45,4 @@ async fn index_tts2_generate() -> Result<()> {
|
|||||||
let i_duration = i_start.elapsed();
|
let i_duration = i_start.elapsed();
|
||||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ fn qwen3_asr_generate() -> Result<()> {
|
|||||||
// RUST_BACKTRACE=1 cargo test -F cuda qwen3_asr_generate -r -- --nocapture
|
// RUST_BACKTRACE=1 cargo test -F cuda qwen3_asr_generate -r -- --nocapture
|
||||||
let save_dir =
|
let save_dir =
|
||||||
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
|
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
|
||||||
let model_path = format!("{}/Qwen/Qwen3-ASR-0.6B/", save_dir); //Qwen/Qwen3-ASR-1.7B
|
let model_path = format!("{}/Qwen/Qwen3-ASR-0.6B/", save_dir); //Qwen/Qwen3-ASR-1.7B
|
||||||
let message = r#"
|
let message = r#"
|
||||||
{
|
{
|
||||||
"model": "qwen3-asr",
|
"model": "qwen3-asr",
|
||||||
|
|||||||
@@ -202,8 +202,8 @@ fn qwen3_weight() -> Result<()> {
|
|||||||
fn index_tts2_weight() -> Result<()> {
|
fn index_tts2_weight() -> Result<()> {
|
||||||
let save_dir: String =
|
let save_dir: String =
|
||||||
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
|
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
|
||||||
let model_path = format!("{}/IndexTeam/IndexTTS-2/", save_dir);
|
let model_path = format!("{}/IndexTeam/IndexTTS-2/", save_dir);
|
||||||
let s2mel_path = model_path+ "/s2mel.pth";
|
let s2mel_path = model_path + "/s2mel.pth";
|
||||||
// let wac2vec2_path = model_path+ "/wav2vec2bert_stats.pt";
|
// let wac2vec2_path = model_path+ "/wav2vec2bert_stats.pt";
|
||||||
// let model_path = format!("{}/iic/speech_campplus_sv_zh-cn_16k-common/", save_dir);
|
// let model_path = format!("{}/iic/speech_campplus_sv_zh-cn_16k-common/", save_dir);
|
||||||
// let campplus_path = model_path+ "/campplus_cn_common.bin";
|
// let campplus_path = model_path+ "/campplus_cn_common.bin";
|
||||||
@@ -229,9 +229,9 @@ fn index_tts2_weight() -> Result<()> {
|
|||||||
// let model_list = vec![semantic_codec_path];
|
// let model_list = vec![semantic_codec_path];
|
||||||
// for m in model_list {
|
// for m in model_list {
|
||||||
// let weights = safetensors::load(m, &device)?;
|
// let weights = safetensors::load(m, &device)?;
|
||||||
// for (key, tensor) in weights.iter() {
|
// for (key, tensor) in weights.iter() {
|
||||||
// println!("=== {} === {:?}", key, tensor.shape());
|
// println!("=== {} === {:?}", key, tensor.shape());
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user