add image api and audio api

This commit is contained in:
jhqxxx
2025-12-25 20:25:52 +08:00
parent bf53d3d869
commit 9a4fa636be
16 changed files with 694 additions and 72 deletions
+12 -10
View File
@@ -1,3 +1,4 @@
use aha::utils::get_default_save_dir;
use anyhow::Result;
use candle_core::Tensor;
@@ -5,18 +6,19 @@ use candle_core::Tensor;
fn messy_test() -> Result<()> {
// RUST_BACKTRACE=1 cargo test -F cuda messy_test -r -- --nocapture
let device = &candle_core::Device::Cpu;
// let path = get_default_save_dir();
let x = Tensor::arange(0.0, 9.0, device)?;
println!("x: {}", x);
let x = x
.unsqueeze(0)?
.unsqueeze(0)?
.broadcast_as((5, 5, 9))?
.reshape((5, 5, 3, 3))?;
println!("x: {}", x);
let x = x.permute((0, 2, 1, 3))?;
println!("x: {}", x);
let x = x.reshape((15, 15))?;
println!("x: {}", x);
// let x = x
// .unsqueeze(0)?
// .unsqueeze(0)?
// .broadcast_as((5, 5, 9))?
// .reshape((5, 5, 3, 3))?;
// println!("x: {}", x);
// let x = x.permute((0, 2, 1, 3))?;
// println!("x: {}", x);
// let x = x.reshape((15, 15))?;
// println!("x: {}", x);
// let xs = Tensor::rand(0.0, 5.0, (1, 1, 3, 3), device)?;
// println!("xs: {}", xs);
// let xs = xs.pad_with_zeros(3, 2, 2)?
+3 -3
View File
@@ -1,6 +1,6 @@
use std::time::Instant;
use aha::models::rmbg2_0::generate::RMBG2_0;
use aha::models::rmbg2_0::generate::RMBG2_0Model;
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
use anyhow::Result;
@@ -31,12 +31,12 @@ fn rmbg2_0_generate() -> Result<()> {
"#;
let mes: ChatCompletionParameters = serde_json::from_str(message)?;
let i_start = Instant::now();
let model = RMBG2_0::init(model_path, None, None)?;
let model = RMBG2_0Model::init(model_path, None, None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
let i_start = Instant::now();
let result = model.generate(mes)?;
let result = model.inference(mes)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in generate is: {:?}", i_duration);
for (i, img) in result.iter().enumerate() {
+57 -7
View File
@@ -1,14 +1,64 @@
use std::time::Instant;
use aha::{
models::voxcpm::{generate::VoxCPMGenerate, tokenizer::SingleChineseTokenizer},
utils::audio_utils::save_wav,
models::{
GenerateModel,
voxcpm::{generate::VoxCPMGenerate, tokenizer::SingleChineseTokenizer},
},
utils::audio_utils::{extract_and_save_audio_from_response, save_wav},
};
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
use anyhow::{Ok, Result};
#[test]
fn voxcpm_use_message_generate() -> Result<()> {
// RUST_BACKTRACE=1 cargo test -F cuda voxcpm_use_message_generate -r -- --nocapture
let model_path = "/home/jhq/huggingface_model/openbmb/VoxCPM-0.5B/";
let message = r#"
{
"model": "voxcpm",
"messages": [
{
"role": "user",
"content": [
{
"type": "audio",
"audio_url":
{
"url": "https://sis-sample-audio.obs.cn-north-1.myhuaweicloud.com/16k16bit.wav"
}
},
{
"type": "text",
"text": "VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly realistic speech."
}
]
}
],
"metadata": {"prompt_text": "华为致力于把数字世界带给每个人,每个家庭,每个组织,构建万物互联的智能世界。"}
}
"#;
let mes: ChatCompletionParameters = serde_json::from_str(message)?;
let i_start = Instant::now();
let mut voxcpm_generate = VoxCPMGenerate::init(model_path, None, None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
let i_start = Instant::now();
let generate = voxcpm_generate.generate(mes)?;
let save_path = extract_and_save_audio_from_response(&generate, "./")?;
for path in save_path {
println!("save audio: {}", path);
}
let i_duration = i_start.elapsed();
println!("Time elapsed in generate is: {:?}", i_duration);
// save_wav(&generate, "voxcpm.wav", 16000)?;
Ok(())
}
#[test]
fn voxcpm_generate() -> Result<()> {
// RUST_BACKTRACE=1 cargo test -F cuda,flash-attn voxcpm_generate -r -- --nocapture
// RUST_BACKTRACE=1 cargo test -F cuda voxcpm_generate -r -- --nocapture
let model_path = "/home/jhq/huggingface_model/openbmb/VoxCPM-0.5B/";
let i_start = Instant::now();
@@ -18,12 +68,12 @@ fn voxcpm_generate() -> Result<()> {
let i_start = Instant::now();
// let generate = voxcpm_generate.generate_simple("太阳当空照,花儿对我笑,小鸟说早早早".to_string())?;
let generate = voxcpm_generate.generate(
let generate = voxcpm_generate.inference(
"VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly realistic speech.".to_string(),
Some("啥子小师叔,打狗还要看主人,你再要继续,我,就是你的对手".to_string()),
Some("./assets/audio/voice_01.wav".to_string()),
Some("file://./assets/audio/voice_01.wav".to_string()),
// Some("一定被灰太狼给吃了,我已经为他准备好了花圈了".to_string()),
// Some("./assets/audio/voice_05.wav".to_string()),
// Some("file://./assets/audio/voice_05.wav".to_string()),
2,
100,
10,
@@ -35,7 +85,7 @@ fn voxcpm_generate() -> Result<()> {
// 创建prompt_cache
// let _ = voxcpm_generate.build_prompt_cache(
// "啥子小师叔,打狗还要看主人,你再要继续,我,就是你的对手".to_string(),
// "./assets/audio/voice_01.wav".to_string(),
// "file://./assets/audio/voice_01.wav".to_string(),
// )?;
// // 使用prompt_cache生成语音
// let generate = voxcpm_generate.generate_use_prompt_cache(
+55 -6
View File
@@ -1,11 +1,60 @@
use std::time::Instant;
use aha::{
models::voxcpm::{generate::VoxCPMGenerate, tokenizer::SingleChineseTokenizer},
utils::audio_utils::save_wav,
models::{
GenerateModel,
voxcpm::{generate::VoxCPMGenerate, tokenizer::SingleChineseTokenizer},
},
utils::audio_utils::{extract_and_save_audio_from_response, save_wav},
};
use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
use anyhow::{Ok, Result};
#[test]
fn voxcpm1_5_use_message_generate() -> Result<()> {
// RUST_BACKTRACE=1 cargo test -F cuda voxcpm1_5_use_message_generate -r -- --nocapture
let model_path = "/home/jhq/huggingface_model/OpenBMB/VoxCPM1.5/";
let message = r#"
{
"model": "voxcpm1.5",
"messages": [
{
"role": "user",
"content": [
{
"type": "audio",
"audio_url":
{
"url": "https://sis-sample-audio.obs.cn-north-1.myhuaweicloud.com/16k16bit.wav"
}
},
{
"type": "text",
"text": "VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly realistic speech."
}
]
}
],
"metadata": {"prompt_text": "华为致力于把数字世界带给每个人,每个家庭,每个组织,构建万物互联的智能世界。"}
}
"#;
let mes: ChatCompletionParameters = serde_json::from_str(message)?;
let i_start = Instant::now();
let mut voxcpm_generate = VoxCPMGenerate::init(model_path, None, None)?;
let i_duration = i_start.elapsed();
println!("Time elapsed in load model is: {:?}", i_duration);
let i_start = Instant::now();
let generate = voxcpm_generate.generate(mes)?;
let save_path = extract_and_save_audio_from_response(&generate, "./")?;
for path in save_path {
println!("save audio: {}", path);
}
let i_duration = i_start.elapsed();
println!("Time elapsed in generate is: {:?}", i_duration);
Ok(())
}
#[test]
fn voxcpm1_5_generate() -> Result<()> {
// RUST_BACKTRACE=1 cargo test -F cuda voxcpm1_5_generate -r -- --nocapture
@@ -18,12 +67,12 @@ fn voxcpm1_5_generate() -> Result<()> {
let i_start = Instant::now();
// let generate = voxcpm_generate.generate_simple("太阳当空照,花儿对我笑,小鸟说早早早".to_string())?;
let generate = voxcpm_generate.generate(
let generate = voxcpm_generate.inference(
"VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly realistic speech.".to_string(),
Some("啥子小师叔,打狗还要看主人,你再要继续,我就是你的对手".to_string()),
Some("./assets/audio/voice_01.wav".to_string()),
Some("file://./assets/audio/voice_01.wav".to_string()),
// Some("一定被灰太狼给吃了,我已经为他准备好了花圈了".to_string()),
// Some("./assets/audio/voice_05.wav".to_string()),
// Some("file://./assets/audio/voice_05.wav".to_string()),
2,
4096,
10,
@@ -35,7 +84,7 @@ fn voxcpm1_5_generate() -> Result<()> {
// 创建prompt_cache
// let _ = voxcpm_generate.build_prompt_cache(
// "啥子小师叔,打狗还要看主人,你再要继续,我,就是你的对手".to_string(),
// "./assets/audio/voice_01.wav".to_string(),
// "file://./assets/audio/voice_01.wav".to_string(),
// )?;
// // 使用prompt_cache生成语音
// let generate = voxcpm_generate.generate_use_prompt_cache(