delete index-ttss
This commit is contained in:
jhqxxx
Generated
-2
@@ -50,10 +50,8 @@ dependencies = [
|
||||
"minijinja",
|
||||
"modelscope",
|
||||
"num",
|
||||
"rand 0.9.2",
|
||||
"rayon",
|
||||
"realfft",
|
||||
"regex",
|
||||
"reqwest 0.12.24",
|
||||
"rocket",
|
||||
"sentencepiece",
|
||||
|
||||
@@ -41,8 +41,6 @@ zip = "7.2.0"
|
||||
half = "2.7.1"
|
||||
byteorder = "1.5.0"
|
||||
sentencepiece = "0.13.1"
|
||||
regex = "1.12.3"
|
||||
rand = "0.9.2"
|
||||
|
||||
[features]
|
||||
flash-attn = ["candle-flash-attn"]
|
||||
|
||||
+3
-236
@@ -2,9 +2,9 @@ use anyhow::{Result, anyhow};
|
||||
use candle_core::{D, IndexOp, Tensor};
|
||||
use candle_nn::{
|
||||
Activation, BatchNorm, BatchNormConfig, Conv1d, Conv1dConfig, Conv2d, Conv2dConfig,
|
||||
ConvTranspose1d, ConvTranspose1dConfig, Embedding, Init, LayerNorm, LayerNormConfig, Linear,
|
||||
Module, ModuleT, RmsNorm, VarBuilder, batch_norm, conv1d, conv1d_no_bias, conv2d,
|
||||
conv2d_no_bias, embedding, layer_norm, linear_b, linear_no_bias, ops::sigmoid, rms_norm,
|
||||
ConvTranspose1d, ConvTranspose1dConfig, Embedding, LayerNorm, LayerNormConfig, Linear, Module,
|
||||
ModuleT, RmsNorm, VarBuilder, batch_norm, conv1d, conv1d_no_bias, conv2d, conv2d_no_bias,
|
||||
embedding, layer_norm, linear_b, linear_no_bias, ops::sigmoid, rms_norm,
|
||||
};
|
||||
use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
|
||||
|
||||
@@ -1194,236 +1194,3 @@ pub fn mish(xs: &Tensor) -> Result<Tensor> {
|
||||
let xs = xs.mul(&tanh)?;
|
||||
Ok(xs)
|
||||
}
|
||||
|
||||
pub struct GPT2Attention {
|
||||
num_heads: usize,
|
||||
head_dim: usize,
|
||||
c_attn: Linear,
|
||||
c_proj: Linear,
|
||||
kv_cache: Option<(Tensor, Tensor)>,
|
||||
}
|
||||
|
||||
impl GPT2Attention {
|
||||
pub fn new(vb: VarBuilder, hidden_size: usize, num_heads: usize) -> Result<Self> {
|
||||
let c_attn_weight = vb
|
||||
.get_with_hints(
|
||||
(hidden_size, 3 * hidden_size),
|
||||
"c_attn.weight",
|
||||
Init::Const(1.0),
|
||||
)?
|
||||
.t()?;
|
||||
let c_attn_bias = vb.get_with_hints(3 * hidden_size, "c_attn.bias", Init::Const(0.0))?;
|
||||
let c_attn = Linear::new(c_attn_weight, Some(c_attn_bias));
|
||||
// let c_attn = linear_b(3 * hidden_size, hidden_size, true, vb.pp("c_attn"))?;
|
||||
let c_proj_weight = vb
|
||||
.get_with_hints(
|
||||
(hidden_size, hidden_size),
|
||||
"c_proj.weight",
|
||||
Init::Const(1.0),
|
||||
)?
|
||||
.t()?;
|
||||
let c_proj_bias = vb.get_with_hints(hidden_size, "c_proj.bias", Init::Const(0.0))?;
|
||||
let c_proj = Linear::new(c_proj_weight, Some(c_proj_bias));
|
||||
// let c_proj = linear_b(hidden_size, hidden_size, true, vb.pp("c_proj"))?;
|
||||
let head_dim = hidden_size / num_heads;
|
||||
Ok(Self {
|
||||
num_heads,
|
||||
head_dim,
|
||||
c_attn,
|
||||
c_proj,
|
||||
kv_cache: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&mut self, xs: &Tensor, attention_mask: Option<&Tensor>) -> Result<Tensor> {
|
||||
let (b, seq_len, _) = xs.dims3()?;
|
||||
let xs = self.c_attn.forward(xs)?;
|
||||
let xs_splits = xs.chunk(3, 2)?;
|
||||
let query_states = xs_splits[0]
|
||||
.as_ref()
|
||||
.reshape((b, seq_len, self.num_heads, self.head_dim))?
|
||||
.transpose(1, 2)?;
|
||||
let key_states = xs_splits[1]
|
||||
.as_ref()
|
||||
.reshape((b, seq_len, self.num_heads, self.head_dim))?
|
||||
.transpose(1, 2)?;
|
||||
let value_states = xs_splits[2]
|
||||
.as_ref()
|
||||
.reshape((b, seq_len, self.num_heads, self.head_dim))?
|
||||
.transpose(1, 2)?;
|
||||
let (key_states, value_states) = match &self.kv_cache {
|
||||
None => (key_states, value_states),
|
||||
Some((prev_k, prev_v)) => {
|
||||
let key_states = Tensor::cat(&[prev_k, &key_states], 2)?;
|
||||
let value_states = Tensor::cat(&[prev_v, &value_states], 2)?;
|
||||
(key_states, value_states)
|
||||
}
|
||||
};
|
||||
|
||||
self.kv_cache = Some((key_states.clone(), value_states.clone()));
|
||||
let scale = 1f64 / f64::sqrt(self.head_dim as f64);
|
||||
let attn_output = eager_attention_forward(
|
||||
&query_states,
|
||||
&key_states,
|
||||
&value_states,
|
||||
None,
|
||||
attention_mask,
|
||||
scale,
|
||||
)?;
|
||||
let attn_output = attn_output.reshape((b, seq_len, self.num_heads * self.head_dim))?;
|
||||
let attn_output = attn_output.apply(&self.c_proj)?;
|
||||
Ok(attn_output)
|
||||
}
|
||||
|
||||
pub fn clear_kv_cache(&mut self) {
|
||||
self.kv_cache = None
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GPT2MLP {
|
||||
linear1: Linear,
|
||||
linear2: Linear,
|
||||
act: Activation,
|
||||
}
|
||||
|
||||
impl GPT2MLP {
|
||||
pub fn new(
|
||||
vb: VarBuilder,
|
||||
in_dim: usize,
|
||||
middle_dim: usize,
|
||||
out_dim: usize,
|
||||
act: Activation,
|
||||
) -> Result<Self> {
|
||||
let c_fc_weight = vb
|
||||
.get_with_hints((in_dim, middle_dim), "c_fc.weight", Init::Const(1.0))?
|
||||
.t()?;
|
||||
let c_fc_bias = vb.get_with_hints(middle_dim, "c_fc.bias", Init::Const(0.0))?;
|
||||
let c_fc = Linear::new(c_fc_weight, Some(c_fc_bias));
|
||||
|
||||
let c_proj_weight = vb
|
||||
.get_with_hints((middle_dim, out_dim), "c_proj.weight", Init::Const(1.0))?
|
||||
.t()?;
|
||||
let c_proj_bias = vb.get_with_hints(out_dim, "c_proj.bias", Init::Const(0.0))?;
|
||||
let c_proj = Linear::new(c_proj_weight, Some(c_proj_bias));
|
||||
|
||||
Ok(Self {
|
||||
linear1: c_fc,
|
||||
linear2: c_proj,
|
||||
act,
|
||||
})
|
||||
}
|
||||
pub fn forward(&self, xs: &Tensor) -> Result<Tensor> {
|
||||
let xs = xs
|
||||
.apply(&self.linear1)?
|
||||
.apply(&self.act)?
|
||||
.apply(&self.linear2)?;
|
||||
Ok(xs)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GPT2Block {
|
||||
ln_1: LayerNorm,
|
||||
attn: GPT2Attention,
|
||||
ln_2: LayerNorm,
|
||||
mlp: GPT2MLP,
|
||||
}
|
||||
|
||||
impl GPT2Block {
|
||||
pub fn new(
|
||||
vb: VarBuilder,
|
||||
hidden_size: usize,
|
||||
num_heads: usize,
|
||||
inner_dim: Option<usize>,
|
||||
) -> Result<Self> {
|
||||
let inner_dim = inner_dim.unwrap_or(4 * hidden_size);
|
||||
let ln_1 = get_layer_norm(vb.pp("ln_1"), 1e-5, hidden_size, true)?;
|
||||
let attn = GPT2Attention::new(vb.pp("attn"), hidden_size, num_heads)?;
|
||||
let ln_2 = get_layer_norm(vb.pp("ln_2"), 1e-5, hidden_size, true)?;
|
||||
let mlp = GPT2MLP::new(
|
||||
vb.pp("mlp"),
|
||||
hidden_size,
|
||||
inner_dim,
|
||||
hidden_size,
|
||||
Activation::NewGelu,
|
||||
)?;
|
||||
Ok(Self {
|
||||
ln_1,
|
||||
attn,
|
||||
ln_2,
|
||||
mlp,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&mut self, xs: &Tensor, attention_mask: Option<&Tensor>) -> Result<Tensor> {
|
||||
let residual = xs.clone();
|
||||
let xs = self.ln_1.forward(xs)?;
|
||||
let xs = self.attn.forward(&xs, attention_mask)?;
|
||||
let residual = xs.add(&residual)?;
|
||||
let xs = self.ln_2.forward(&residual)?;
|
||||
let xs = self.mlp.forward(&xs)?;
|
||||
let xs = xs.add(&residual)?;
|
||||
Ok(xs)
|
||||
}
|
||||
pub fn clear_kv_cache(&mut self) {
|
||||
self.attn.clear_kv_cache()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub struct GPT2Model {
|
||||
wte: Embedding,
|
||||
// wpe: Embedding,
|
||||
h: Vec<GPT2Block>,
|
||||
ln_f: LayerNorm,
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
impl GPT2Model {
|
||||
pub fn new(
|
||||
vb: VarBuilder,
|
||||
hidden_size: usize,
|
||||
num_heads: usize,
|
||||
num_hidden_layers: usize,
|
||||
wte_embeddings: &Tensor,
|
||||
) -> Result<Self> {
|
||||
// let wte = embedding(vocab_size, hidden_size, vb.pp("wte"))?;
|
||||
let wte = Embedding::new(wte_embeddings.clone(), hidden_size);
|
||||
// let wpe = embedding(max_position_embeddings, hidden_size, vb.pp("wpe"))?;
|
||||
let vb_layers = vb.pp("h");
|
||||
let mut h = vec![];
|
||||
for i in 0..num_hidden_layers {
|
||||
let block = GPT2Block::new(vb_layers.pp(i), hidden_size, num_heads, None)?;
|
||||
h.push(block);
|
||||
}
|
||||
let ln_f = get_layer_norm(vb.pp("ln_f"), 1e-5, hidden_size, true)?;
|
||||
Ok(Self { wte, h, ln_f })
|
||||
}
|
||||
|
||||
pub fn forward(&mut self, inputs_embeds: &Tensor) -> Result<Tensor> {
|
||||
let (b_size, seq_len, _) = inputs_embeds.dims3()?;
|
||||
let mut xs = inputs_embeds.clone();
|
||||
let attention_mask: Option<Tensor> = {
|
||||
if seq_len <= 1 {
|
||||
None
|
||||
} else {
|
||||
Some(prepare_causal_attention_mask(
|
||||
b_size,
|
||||
seq_len,
|
||||
0,
|
||||
xs.device(),
|
||||
)?)
|
||||
}
|
||||
};
|
||||
for block in &mut self.h {
|
||||
xs = block.forward(&xs, attention_mask.as_ref())?;
|
||||
}
|
||||
xs = self.ln_f.forward(&xs)?;
|
||||
Ok(xs)
|
||||
}
|
||||
|
||||
pub fn clear_kv_cache(&mut self) {
|
||||
for layer in self.h.iter_mut() {
|
||||
layer.clear_kv_cache()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,228 +0,0 @@
|
||||
use serde::{Deserialize, Deserializer};
|
||||
|
||||
use crate::models::mask_gct::config::SemanticCodec;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct IndexTTS2Config {
|
||||
pub dataset: Dataset,
|
||||
pub gpt: GptConfig,
|
||||
pub semantic_codec: SemanticCodec,
|
||||
pub s2mel: S2MelConfig,
|
||||
pub gpt_checkpoint: String,
|
||||
pub w2v_stat: String,
|
||||
pub s2mel_checkpoint: String,
|
||||
pub emo_matrix: String,
|
||||
pub spk_matrix: String,
|
||||
pub emo_num: Vec<usize>,
|
||||
pub qwen_emo_path: String,
|
||||
pub vocoder: Vocoder,
|
||||
pub version: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct Dataset {
|
||||
pub bpe_model: String,
|
||||
pub sample_rate: usize,
|
||||
pub squeeze: bool,
|
||||
pub mel: Mel,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct Mel {
|
||||
pub sample_rate: usize,
|
||||
pub n_fft: usize,
|
||||
pub hop_length: usize,
|
||||
pub win_length: usize,
|
||||
pub n_mels: usize,
|
||||
pub mel_fmin: usize,
|
||||
pub normalize: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct GptConfig {
|
||||
pub model_dim: usize,
|
||||
pub max_mel_tokens: usize,
|
||||
pub max_text_tokens: usize,
|
||||
pub heads: usize,
|
||||
pub use_mel_codes_as_input: bool,
|
||||
pub mel_length_compression: usize,
|
||||
pub layers: usize,
|
||||
pub number_text_tokens: usize,
|
||||
pub number_mel_codes: usize,
|
||||
pub start_mel_token: usize,
|
||||
pub stop_mel_token: usize,
|
||||
pub start_text_token: usize,
|
||||
pub stop_text_token: usize,
|
||||
pub train_solo_embeddings: bool,
|
||||
pub condition_type: String,
|
||||
pub condition_module: ConditionModule,
|
||||
pub emo_condition_module: EmoConditionModule,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct ConditionModule {
|
||||
pub output_size: usize,
|
||||
pub linear_units: usize,
|
||||
pub attention_heads: usize,
|
||||
pub num_blocks: usize,
|
||||
pub input_layer: String,
|
||||
pub perceiver_mult: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct EmoConditionModule {
|
||||
pub output_size: usize,
|
||||
pub linear_units: usize,
|
||||
pub attention_heads: usize,
|
||||
pub num_blocks: usize,
|
||||
pub input_layer: String,
|
||||
pub perceiver_mult: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct S2MelConfig {
|
||||
pub preprocess_params: PreprocessParams,
|
||||
pub dit_type: String,
|
||||
pub reg_loss_type: String,
|
||||
pub style_encoder: StyleEncoder,
|
||||
pub length_regulator: LengthRegulator,
|
||||
#[serde(rename = "DiT")]
|
||||
pub di_t: DiTConfig,
|
||||
pub wavenet: WavenetConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct PreprocessParams {
|
||||
pub sr: usize,
|
||||
pub spect_params: SpectParams,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct SpectParams {
|
||||
pub n_fft: usize,
|
||||
pub win_length: usize,
|
||||
pub hop_length: usize,
|
||||
pub n_mels: usize,
|
||||
pub fmin: usize,
|
||||
#[serde(deserialize_with = "deserialize_optional_fmax")]
|
||||
pub fmax: Option<usize>,
|
||||
}
|
||||
|
||||
fn deserialize_optional_fmax<'de, D>(deserializer: D) -> Result<Option<usize>, D::Error>
|
||||
where D: Deserializer<'de> {
|
||||
let opt: Option<serde_json::Value> = Option::deserialize(deserializer)?;
|
||||
match opt {
|
||||
Some(serde_json::Value::String(s)) if s == "None" || s == "null" => Ok(None),
|
||||
Some(serde_json::Value::Number(n)) => {
|
||||
if let Some(n) = n.as_u64() {
|
||||
Ok(Some(n as usize))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("Expected positive integer"))
|
||||
}
|
||||
}
|
||||
Some(_) => Err(serde::de::Error::custom("Expected number or 'None' string")),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct StyleEncoder {
|
||||
pub dim: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct LengthRegulator {
|
||||
pub channels: usize,
|
||||
pub is_discrete: bool,
|
||||
pub in_channels: usize,
|
||||
pub content_codebook_size: usize,
|
||||
pub sampling_ratios: Vec<usize>,
|
||||
pub vector_quantize: bool,
|
||||
pub n_codebooks: usize,
|
||||
pub quantizer_dropout: f32,
|
||||
pub f0_condition: bool,
|
||||
pub n_f0_bins: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct DiTConfig {
|
||||
pub hidden_dim: usize,
|
||||
pub num_heads: usize,
|
||||
pub depth: usize,
|
||||
pub class_dropout_prob: f32,
|
||||
pub block_size: usize,
|
||||
pub in_channels: usize,
|
||||
pub style_condition: bool,
|
||||
pub final_layer_type: String,
|
||||
pub target: String,
|
||||
pub content_dim: usize,
|
||||
pub content_codebook_size: usize,
|
||||
pub content_type: String,
|
||||
pub f0_condition: bool,
|
||||
pub n_f0_bins: usize,
|
||||
pub content_codebooks: usize,
|
||||
pub is_causal: bool,
|
||||
pub long_skip_connection: bool,
|
||||
pub zero_prompt_speech_token: bool,
|
||||
pub time_as_token: bool,
|
||||
pub style_as_token: bool,
|
||||
pub uvit_skip_connection: bool,
|
||||
pub add_resblock_in_transformer: bool,
|
||||
}
|
||||
|
||||
pub struct DiTModelArgs {
|
||||
pub block_size: usize,
|
||||
pub vocab_size: usize,
|
||||
pub n_layer: usize,
|
||||
pub n_head: usize,
|
||||
pub dim: usize,
|
||||
pub intermediate_size: usize,
|
||||
pub n_local_heads: usize,
|
||||
pub head_dim: usize,
|
||||
pub rope_base: f32,
|
||||
pub norm_eps: f64,
|
||||
pub has_cross_attention: bool,
|
||||
pub context_dim: usize,
|
||||
pub uvit_skip_connection: bool,
|
||||
pub time_as_token: bool,
|
||||
}
|
||||
|
||||
impl DiTModelArgs {
|
||||
pub fn new_from_dit_config(config: &DiTConfig) -> Self {
|
||||
let hidden_dim = 4 * config.hidden_dim;
|
||||
let n_hidden = 2 * hidden_dim / 3;
|
||||
let intermediate_size = n_hidden + (256 - n_hidden % 256) % 256;
|
||||
Self {
|
||||
block_size: config.block_size,
|
||||
vocab_size: 1024,
|
||||
n_layer: config.depth,
|
||||
n_head: config.num_heads,
|
||||
dim: config.hidden_dim,
|
||||
intermediate_size,
|
||||
n_local_heads: config.num_heads,
|
||||
head_dim: config.hidden_dim / config.num_heads,
|
||||
rope_base: 10000.0,
|
||||
norm_eps: 1e-5,
|
||||
has_cross_attention: false,
|
||||
context_dim: 0,
|
||||
uvit_skip_connection: config.uvit_skip_connection,
|
||||
time_as_token: config.time_as_token,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct WavenetConfig {
|
||||
pub hidden_dim: usize,
|
||||
pub num_layers: usize,
|
||||
pub kernel_size: usize,
|
||||
pub dilation_rate: usize,
|
||||
pub p_dropout: f32,
|
||||
pub style_condition: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
pub struct Vocoder {
|
||||
pub r#type: String,
|
||||
pub name: String,
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
use aha_openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatCompletionResponse};
|
||||
use anyhow::{Result, anyhow};
|
||||
use base64::{Engine, prelude::BASE64_STANDARD};
|
||||
use candle_core::{DType, Device};
|
||||
use sentencepiece::SentencePieceProcessor;
|
||||
|
||||
use crate::{
|
||||
models::index_tts2::{
|
||||
config::IndexTTS2Config, model::IndexTTS2Model, utils::tokenize_by_cjk_char,
|
||||
},
|
||||
tokenizer::sentencepiece_encode,
|
||||
utils::{
|
||||
audio_utils::get_audio_wav_u8, build_audio_completion_response, extract_user_text,
|
||||
get_default_save_dir, get_device,
|
||||
},
|
||||
};
|
||||
|
||||
pub struct IndexTTS2Generate {
|
||||
tokenizer: SentencePieceProcessor,
|
||||
// config: IndexTTS2Config,
|
||||
model: IndexTTS2Model,
|
||||
device: Device,
|
||||
sample_rate: u32,
|
||||
model_name: String,
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
impl IndexTTS2Generate {
|
||||
pub fn init(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self> {
|
||||
let config_path = path.to_string() + "/config.yaml";
|
||||
let save_dir = get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
|
||||
let config: IndexTTS2Config = serde_yaml::from_slice(&std::fs::read(config_path)?)?;
|
||||
let device = get_device(device);
|
||||
let bpe_path = path.to_string() + "/bpe.model";
|
||||
let tokenizer = SentencePieceProcessor::open(bpe_path)
|
||||
.map_err(|e| anyhow!(format!("load bpe,model file error:{}", e)))?;
|
||||
let model = IndexTTS2Model::new(path, &save_dir, &config, &device)?;
|
||||
Ok(Self {
|
||||
tokenizer,
|
||||
// config,
|
||||
model,
|
||||
device,
|
||||
sample_rate: 22050,
|
||||
model_name: "index-tts2".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
|
||||
let text = extract_user_text(&mes)?;
|
||||
let text = tokenize_by_cjk_char(&text, true);
|
||||
let input_ids = sentencepiece_encode(&text, &self.tokenizer, &self.device)?;
|
||||
|
||||
let audio = self.model.forward(&input_ids, &mes)?;
|
||||
let wav_u8 = get_audio_wav_u8(&audio, self.sample_rate)?;
|
||||
let base64_audio = BASE64_STANDARD.encode(wav_u8);
|
||||
let response = build_audio_completion_response(&base64_audio, &self.model_name);
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
pub mod config;
|
||||
pub mod generate;
|
||||
pub mod model;
|
||||
// pub mod processor;
|
||||
pub mod utils;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,172 +0,0 @@
|
||||
use anyhow::{Result, anyhow};
|
||||
use regex::Regex;
|
||||
|
||||
use crate::utils::{download_model, get_default_save_dir};
|
||||
|
||||
pub async fn download_index_tts2_need_model(save_dir: Option<&str>) -> Result<()> {
|
||||
let save_dir = match save_dir {
|
||||
Some(dir) => dir.to_string(),
|
||||
None => get_default_save_dir().expect("Failed to get home directory"),
|
||||
};
|
||||
|
||||
let w2v_bert2_0 = "facebook/w2v-bert-2.0";
|
||||
let mask_gct = "amphion/MaskGCT";
|
||||
// let campplus= "funasr/campplus"; // huggingface
|
||||
let campplus = "iic/speech_campplus_sv_zh-cn_16k-common"; // modelscope
|
||||
// let bigvgan = "nvidia/bigvgan_v2_22khz_80band_256x"; // huggingface
|
||||
let bigvgan = "nv-community/bigvgan_v2_22khz_80band_256x"; // modelscope
|
||||
download_model(w2v_bert2_0, &save_dir, 3).await?;
|
||||
download_model(mask_gct, &save_dir, 3).await?;
|
||||
download_model(campplus, &save_dir, 3).await?;
|
||||
download_model(bigvgan, &save_dir, 3).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TextNormalizer {
|
||||
// char_rep_map: HashMap<String, String>,
|
||||
// zh_char_rep_map: HashMap<String, String>,
|
||||
pinyin_tone_pattern: Regex,
|
||||
// name_pattern: Regex,
|
||||
// tech_term_pattern: Regex,
|
||||
// english_contraction_pattern: Regex,
|
||||
email_pattern: Regex,
|
||||
cjk_range_pattern: Regex,
|
||||
}
|
||||
|
||||
impl TextNormalizer {
|
||||
pub fn new() -> Result<Self> {
|
||||
let pinyin_tone_pattern: Regex = Regex::new(
|
||||
// r"(?i)(?<![a-z])((?:[bpmfdtnlgkhjqxzcsryw]|[zcs]h)?(?:[aeiouüvAEIOUV]|[aeAE]i|u[aiouAIUO]|aoAO|ouOU|i[aeuAEU]|[uüvUÜV]e|[uvüUVÜ]ang?|uaiUAI|[aeiuvAEIUV]n|[aeioAEIO]ng|ia[noNAO]|i[aA][oO]ng)|ngNG|erER)([1-5])"
|
||||
r"(?i)((?:[bpmfdtnlgkhjqxzcsryw]|[zcs]h)?(?:[aeiouüvAEIOUV]|[aeAE]i|u[aiouAIUO]|aoAO|ouOU|i[aeuAEU]|[uüvUÜV]e|[uvüUVÜ]ang?|uaiUAI|[aeiuvAEIUV]n|[aeioAEIO]ng|ia[noNAO]|i[aA][oO]ng)|ngNG|erER)([1-5])"
|
||||
).map_err(|e| anyhow!(format!("new pinyin_tone_pattern regex error:{}", e)))?;
|
||||
// let name_pattern: Regex =
|
||||
// Regex::new(r"[\u{4e00}-\u{9fff}]+(?:[-·—][\u{4e00}-\u{9fff}]+){1,2}")
|
||||
// .map_err(|e| anyhow!(format!("new name_pattern regex error:{}", e)))?;
|
||||
// let tech_term_pattern: Regex = Regex::new(r"[A-Za-z][A-Za-z0-9]*(?:-[A-Za-z0-9]+)+")
|
||||
// .map_err(|e| anyhow!(format!("new tech_term_pattern regex error:{}", e)))?;
|
||||
// let english_contraction_pattern: Regex = Regex::new(
|
||||
// r"(?i)(what|where|who|which|how|t?here|it|s?he|that|this)'s",
|
||||
// )
|
||||
// .map_err(|e| anyhow!(format!("new english_contraction_pattern regex error:{}", e)))?;
|
||||
let email_pattern: Regex = Regex::new(r"^[a-zA-Z0-9]+@[a-zA-Z0-9]+\.[a-zA-Z]+$")
|
||||
.map_err(|e| anyhow!(format!("new email_pattern regex error:{}", e)))?;
|
||||
let cjk_range_pattern: Regex = Regex::new(r"([\u{1100}-\u{11ff}\u{2e80}-\u{a4cf}\u{a840}-\u{d7af}\u{f900}-\u{faff}\u{fe30}-\u{fe4f}\u{ff65}-\u{ffdc}\u{20000}-\u{2ffff}])")
|
||||
.map_err(|e| anyhow!(format!("new cjk_range_pattern regex error:{}", e)))?;
|
||||
// let mut char_rep_map = HashMap::new();
|
||||
// char_rep_map.insert("：".to_string(), ",".to_string());
|
||||
// char_rep_map.insert("；".to_string(), ",".to_string());
|
||||
// char_rep_map.insert(";".to_string(), ",".to_string());
|
||||
// char_rep_map.insert("，".to_string(), ",".to_string());
|
||||
// char_rep_map.insert("。".to_string(), ".".to_string());
|
||||
// char_rep_map.insert("！".to_string(), "!".to_string());
|
||||
// char_rep_map.insert("？".to_string(), "?".to_string());
|
||||
// char_rep_map.insert("\n".to_string(), " ".to_string());
|
||||
// char_rep_map.insert("·".to_string(), "-".to_string());
|
||||
// char_rep_map.insert("、".to_string(), ",".to_string());
|
||||
// char_rep_map.insert("...".to_string(), "…".to_string());
|
||||
// char_rep_map.insert(",,,".to_string(), "…".to_string());
|
||||
// char_rep_map.insert("，，，".to_string(), "…".to_string());
|
||||
// char_rep_map.insert("……".to_string(), "…".to_string());
|
||||
// char_rep_map.insert("“".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("”".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("\"".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("‘".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("’".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("（".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("）".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("(".to_string(), "'".to_string());
|
||||
// char_rep_map.insert(")".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("《".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("》".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("【".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("】".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("[".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("]".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("—".to_string(), "-".to_string());
|
||||
// char_rep_map.insert("～".to_string(), "-".to_string());
|
||||
// char_rep_map.insert("~".to_string(), "-".to_string());
|
||||
// char_rep_map.insert("「".to_string(), "'".to_string());
|
||||
// char_rep_map.insert("」".to_string(), "'".to_string());
|
||||
// char_rep_map.insert(":".to_string(), ",".to_string());
|
||||
|
||||
// let mut zh_char_rep_map = char_rep_map.clone();
|
||||
// zh_char_rep_map.insert("$".to_string(), ".".to_string());
|
||||
Ok(Self {
|
||||
// char_rep_map,
|
||||
// zh_char_rep_map,
|
||||
pinyin_tone_pattern,
|
||||
// name_pattern,
|
||||
// tech_term_pattern,
|
||||
// english_contraction_pattern,
|
||||
email_pattern,
|
||||
cjk_range_pattern,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn match_email(&self, email: &str) -> bool {
|
||||
self.email_pattern.is_match(email)
|
||||
}
|
||||
|
||||
pub fn use_chinese(&self, s: &str) -> bool {
|
||||
let has_chinese = s.chars().any(|c| ('\u{4e00}'..='\u{9fff}').contains(&c));
|
||||
let has_alpha = s.chars().any(|c| c.is_alphabetic());
|
||||
let is_email = self.match_email(s);
|
||||
|
||||
if has_chinese || !has_alpha || is_email {
|
||||
return true;
|
||||
}
|
||||
|
||||
self.pinyin_tone_pattern.is_match(s)
|
||||
}
|
||||
|
||||
pub fn tokenize_by_cjk_char(&self, line: &str, do_upper_case: bool) -> String {
|
||||
// Split the line by CJK characters
|
||||
let parts: Vec<&str> = self.cjk_range_pattern.split(line.trim()).collect();
|
||||
// Process each part and join with spaces
|
||||
let mut result_parts = Vec::new();
|
||||
for part in parts {
|
||||
if !part.trim().is_empty() {
|
||||
if do_upper_case {
|
||||
result_parts.push(part.trim().to_uppercase());
|
||||
} else {
|
||||
result_parts.push(part.trim().to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
// Join the parts with spaces
|
||||
result_parts.join(" ")
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tokenize_by_cjk_char(line: &str, do_upper_case: bool) -> String {
|
||||
let mut result_parts = Vec::new();
|
||||
for ch in line.chars() {
|
||||
if ('\u{1100}'..='\u{11ff}').contains(&ch)
|
||||
|| ('\u{2e80}'..='\u{a4cf}').contains(&ch)
|
||||
|| ('\u{a840}'..='\u{d7af}').contains(&ch)
|
||||
|| ('\u{f900}'..='\u{faff}').contains(&ch)
|
||||
|| ('\u{fe30}'..='\u{fe4f}').contains(&ch)
|
||||
|| ('\u{ff65}'..='\u{ffdc}').contains(&ch)
|
||||
|| ('\u{20000}'..='\u{2ffff}').contains(&ch)
|
||||
|| ('\u{4e00}'..='\u{9fff}').contains(&ch)
|
||||
{
|
||||
// CJK 字符
|
||||
if do_upper_case {
|
||||
result_parts.push(ch.to_uppercase().collect::<String>());
|
||||
} else {
|
||||
result_parts.push(ch.to_string());
|
||||
}
|
||||
} else {
|
||||
// 非 CJK 字符，保持在一起
|
||||
if do_upper_case {
|
||||
result_parts.push(ch.to_uppercase().to_string());
|
||||
} else {
|
||||
result_parts.push(ch.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result_parts.join(" ")
|
||||
}
|
||||
@@ -6,7 +6,6 @@ pub mod feature_extractor;
|
||||
pub mod fun_asr_nano;
|
||||
pub mod glm_asr_nano;
|
||||
pub mod hunyuan_ocr;
|
||||
pub mod index_tts2;
|
||||
pub mod mask_gct;
|
||||
pub mod minicpm4;
|
||||
pub mod paddleocr_vl;
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use aha::{
|
||||
models::index_tts2::{generate::IndexTTS2Generate, utils::download_index_tts2_need_model},
|
||||
utils::audio_utils::extract_and_save_audio_from_response,
|
||||
};
|
||||
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
|
||||
use anyhow::Result;
|
||||
|
||||
#[tokio::test]
|
||||
async fn index_tts2_generate() -> Result<()> {
|
||||
// RUST_BACKTRACE=1 cargo test -F cuda index_tts2_generate -r -- --nocapture
|
||||
let save_dir =
|
||||
aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
|
||||
let _ = download_index_tts2_need_model(Some(&save_dir)).await?;
|
||||
let model_path = format!("{}/IndexTeam/IndexTTS-2", save_dir);
|
||||
let message = r#"
|
||||
{
|
||||
"model": "index-tts2",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "audio",
|
||||
"audio_url":
|
||||
{
|
||||
"url": "file:///home/jhq/Videos/voice_01.wav"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "你好啊，吃饭了吗"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
"#;
|
||||
let mes: ChatCompletionParameters = serde_json::from_str(message)?;
|
||||
let i_start = Instant::now();
|
||||
let mut voxcpm_generate = IndexTTS2Generate::init(&model_path, None, None)?;
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in load model is: {:?}", i_duration);
|
||||
|
||||
let i_start = Instant::now();
|
||||
let generate = voxcpm_generate.generate(mes)?;
|
||||
let save_path = extract_and_save_audio_from_response(&generate, "./")?;
|
||||
for path in save_path {
|
||||
println!("save audio: {}", path);
|
||||
}
|
||||
let i_duration = i_start.elapsed();
|
||||
println!("Time elapsed in generate is: {:?}", i_duration);
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user