aha/tests/test_voxcpm.rs

use anyhow::{Ok, Result};
use std::collections::HashMap;

use aha::{
    models::voxcpm::{
        audio_vae::AudioVAE, config::VoxCPMConfig, model::VoxCPMModel,
        tokenizer::SingleChineseTokenizer,
    },
    utils::{
        audio_utils::save_wav,
        utils::{find_type_files, get_device},
    },
};
use candle_core::pickle::read_all_with_key;
use candle_nn::VarBuilder;

#[test]
fn voxcpm_generate() -> Result<()> {
    // RUST_BACKTRACE=1 cargo test -F cuda,flash-attn voxcpm_generate -- --nocapture
    let model_path = "/home/jhq/huggingface_model/openbmb/VoxCPM-0.5B/";
    let model_list = find_type_files(&model_path, "pth")?;
    println!(" pth model_list: {:?}", model_list);
    let dev = get_device(None);
    let mut dict_to_hashmap = HashMap::new();
    let mut dtype = candle_core::DType::F32;
    for m in model_list {
        let dict = read_all_with_key(m, Some("state_dict"))?;
        dtype = dict[0].1.dtype();
        for (k, v) in dict {
            // println!("key: {}, tensor shape: {:?}", k, v);
            // if k.contains("decoder.model.2.block.1") {
            //     println!("val: {}", v);
            // }
            dict_to_hashmap.insert(k, v);
        }
    }
    let vb = VarBuilder::from_tensors(dict_to_hashmap, dtype, &dev);
    let audio_vae = AudioVAE::new(
        vb,
        128,
        vec![2, 5, 8, 8],
        Some(64),
        1536,
        vec![8, 8, 5, 2],
        16000,
    )?;
    println!("audio vae load down");
    let model_list = find_type_files(&model_path, "bin")?;
    println!(" bin model_list: {:?}", model_list);
    dict_to_hashmap = HashMap::new();
    for m in model_list {
        let dict = read_all_with_key(m, Some("state_dict"))?;
        dtype = dict[0].1.dtype();
        for (k, v) in dict {
            // println!("key: {}, tensor shape: {:?}", k, v);
            dict_to_hashmap.insert(k, v);
        }
    }
    let vb_vox = VarBuilder::from_tensors(dict_to_hashmap, dtype, &dev);
    let config_path = model_path.to_string() + "/config.json";
    let config: VoxCPMConfig = serde_json::from_slice(&std::fs::read(config_path)?)?;
    let tokenizer = SingleChineseTokenizer::new(model_path)?;
    let mut voxcpm = VoxCPMModel::new(vb_vox, config, tokenizer, audio_vae)?;
    let generate = voxcpm.generate(
        "太阳当空照，花儿对我笑，小鸟说早早早".to_string(),
        Some("啥子小师叔，打狗还要看主人，你再要继续，我，就是你的对手".to_string()),
        Some("./assets/audio/voice_01.wav".to_string()),
        // Some("一定被灰太狼给吃了，我已经为他准备好了花圈了".to_string()),
        // Some("./assets/audio/voice_05.wav".to_string()),
        2,
        100,
        10,
        2.0,
        false,
        3,
        6.0,
    )?;
    let _ = save_wav(&generate, "voxcpm_init.wav")?;
    Ok(())
}

#[test]
fn voxcpm_tokenizer() -> Result<()> {
    let model_path = "/home/jhq/huggingface_model/openbmb/VoxCPM-0.5B/";
    let tokenizer = SingleChineseTokenizer::new(model_path)?;
    let ids = tokenizer.encode("你好啊，你吃饭了吗".to_string())?;
    println!("ids: {:?}", ids);
    Ok(())
}