tckm.yan/aha
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unify cargo version and add some ci rules

Browse Source
This commit is contained in:
Yijun Zhao
2025-10-15 21:03:49 +08:00
parent 0fd3c7d935
commit 9b9a8f2c73
40 changed files with 873 additions and 836 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/minicpm4/config.rs
+1 -1
View File
@@ -26,4 +26,4 @@ pub struct MiniCPM4Config {
pub scale_emb: f64,
pub dim_model_base: usize,
pub scale_depth: f32,
}
}
src/models/minicpm4/generate.rs
+14 -22
View File
@@ -1,13 +1,3 @@
use crate::models::minicpm4::config::MiniCPM4Config;
use crate::models::minicpm4::model::MiniCPMModel;
// use crate::models::GenerateStream;
use crate::utils::utils::{
build_completion_chunk_response, build_completion_response, find_type_files, get_device, get_dtype, get_logit_processor
};
use crate::{
chat_template::chat_template::ChatTemplate, models::GenerateModel,
tokenizer::tokenizer::TokenizerModel,
};
use anyhow::{Result, anyhow};
use candle_core::{DType, Device, Tensor};
use candle_nn::VarBuilder;
@@ -17,6 +7,15 @@ use openai_dive::v1::resources::chat::{
use rocket::async_stream::stream;
use rocket::futures::Stream;
use crate::models::minicpm4::config::MiniCPM4Config;
use crate::models::minicpm4::model::MiniCPMModel;
// use crate::models::GenerateStream;
use crate::utils::{
build_completion_chunk_response, build_completion_response, find_type_files, get_device,
get_dtype, get_logit_processor,
};
use crate::{chat_template::ChatTemplate, models::GenerateModel, tokenizer::TokenizerModel};
pub struct MiniCPMGenerateModel<'a> {
chat_template: ChatTemplate<'a>,
tokenizer: TokenizerModel,
@@ -26,7 +25,7 @@ pub struct MiniCPMGenerateModel<'a> {
im_end_id: u32,
}
impl <'a> MiniCPMGenerateModel<'a> {
impl<'a> MiniCPMGenerateModel<'a> {
pub fn init(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self> {
let chat_template = ChatTemplate::init(path)?;
let tokenizer = TokenizerModel::init(path)?;
@@ -37,7 +36,7 @@ impl <'a> MiniCPMGenerateModel<'a> {
let dtype = get_dtype(dtype, cfg_dtype);
let endoftext_id = cfg.eos_token_id[0];
let im_end_id = cfg.eos_token_id[1];
let model_list = find_type_files(&path, "safetensors")?;
let model_list = find_type_files(path, "safetensors")?;
let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, device)? };
let minicpm = MiniCPMModel::new(vb, cfg)?;
@@ -53,7 +52,6 @@ impl <'a> MiniCPMGenerateModel<'a> {
}
impl<'a> GenerateModel for MiniCPMGenerateModel<'a> {
fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p);
let mes_render = self.chat_template.apply_chat_template(&mes)?;
@@ -61,10 +59,7 @@ impl<'a> GenerateModel for MiniCPMGenerateModel<'a> {
let mut seq_len = input_ids.dim(1)?;
let mut seqlen_offset = 0;
let mut generate = Vec::new();
let sample_len = match mes.max_tokens {
Some(max) => max,
None => 2048,
};
let sample_len = mes.max_tokens.unwrap_or(2048);
for _ in 0..sample_len {
let logits = self.minicpm.forward_with_cache(&input_ids, seqlen_offset)?;
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
@@ -91,10 +86,7 @@ impl<'a> GenerateModel for MiniCPMGenerateModel<'a> {
let mut input_ids = self.tokenizer.text_encode(mes_render, &self.device)?;
let mut seq_len = input_ids.dim(1)?;
let mut seqlen_offset = 0;
let sample_len = match mes.max_tokens {
Some(max) => max,
None => 512,
};
let sample_len = mes.max_tokens.unwrap_or(512);
let stream = stream! {
let mut error_tokens = Vec::new();
for _ in 0..sample_len {
@@ -105,7 +97,7 @@ impl<'a> GenerateModel for MiniCPMGenerateModel<'a> {
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
let next_token = logit_processor.sample(&logits)?;
let mut decode_ids = Vec::new();
if error_tokens.len() > 0 {
if !error_tokens.is_empty(){
decode_ids.extend_from_slice(&error_tokens);
}
decode_ids.push(next_token);
src/models/minicpm4/mod.rs
+1 -1
View File
@@ -1,3 +1,3 @@
pub mod config;
pub mod generate;
pub mod model;
pub mod generate;
src/models/minicpm4/model.rs
+18 -16
View File
@@ -1,3 +1,7 @@
use anyhow::{Ok, Result};
use candle_core::{D, Device, Tensor};
use candle_nn::{Embedding, Linear, Module, RmsNorm, VarBuilder, embedding, rms_norm};
use crate::{
models::{
common::{AttentionNobias, MLPNoBias},
@@ -6,9 +10,6 @@ use crate::{
position_embed::rope::compute_default_rope_parameters,
utils::tensor_utils::prepare_causal_attention_mask,
};
use anyhow::{Ok, Result};
use candle_core::{D, Device, Tensor};
use candle_nn::{Embedding, Linear, Module, RmsNorm, VarBuilder, embedding, rms_norm};
pub struct MiniCPMLongRoPE {
short_factor: Vec<f32>,
@@ -33,15 +34,13 @@ impl MiniCPMLongRoPE {
let scaling_factor =
(1.0 + scale.ln() / (original_max_position_embeddings as f64).ln()).sqrt();
let inv_freq = compute_default_rope_parameters(head_dim, rope_theta);
let inv_freq =
Tensor::from_slice(&inv_freq, (1, inv_freq.len()), device)?;
let inv_freq = Tensor::from_slice(&inv_freq, (1, inv_freq.len()), device)?;
let max_seq_len_cached = max_position_embeddings;
let t = Tensor::arange(0.0_f32, max_position_embeddings as f32, device)?
.reshape((max_position_embeddings, 1))?;
// short_factor.len() = 32
// head_dim = 1024 / 16 = 64, inv_freq.len() = 32
let ext_factors =
Tensor::from_slice(&short_factor, (1, short_factor.len()), device)?;
let ext_factors = Tensor::from_slice(&short_factor, (1, short_factor.len()), device)?;
let ext_factors = Tensor::ones_like(&ext_factors)?.div(&ext_factors)?;
// (seq_len, 1) matmul (1, 32) -> (seq_len, 32) * (1, 32)-> (seq_len, 32)
let freqs = t.matmul(&ext_factors)?.broadcast_mul(&inv_freq)?;
@@ -63,8 +62,7 @@ impl MiniCPMLongRoPE {
}
pub fn update_cos_sin_cache(&mut self, seqlen: usize) -> Result<()> {
self.max_seq_len_cached = seqlen;
let t = Tensor::arange(0.0_f32, seqlen as f32, &self.device)?
.reshape((seqlen, 1))?;
let t = Tensor::arange(0.0_f32, seqlen as f32, &self.device)?.reshape((seqlen, 1))?;
let mut ext_factors = Tensor::from_slice(
&self.short_factor,
(1, self.short_factor.len()),
@@ -85,7 +83,7 @@ impl MiniCPMLongRoPE {
}
pub fn forward(&mut self, pos_offset: usize, seqlen: usize) -> Result<(Tensor, Tensor)> {
if pos_offset + seqlen > self.max_seq_len_cached {
let _ = self.update_cos_sin_cache(pos_offset + seqlen)?;
self.update_cos_sin_cache(pos_offset + seqlen)?;
}
let cos = self.cos_cached.narrow(0, pos_offset, seqlen)?;
let sin = self.sin_cached.narrow(0, pos_offset, seqlen)?;
@@ -143,7 +141,9 @@ impl MiniCPMDecoderLayer {
) -> Result<Tensor> {
let residual = xs.clone();
let xs = self.input_layernorm.forward(xs)?;
let xs = self.self_attn.forward(&xs, cos, sin, attention_mask, true)?;
let xs = self
.self_attn
.forward(&xs, cos, sin, attention_mask, true)?;
let xs = (residual
+ xs.affine(
self.scale_depth as f64 / (self.num_hidden_layers as f64).sqrt(),
@@ -168,7 +168,9 @@ impl MiniCPMDecoderLayer {
) -> Result<Tensor> {
let residual = xs.clone();
let xs = self.input_layernorm.forward(xs)?;
let xs = self.self_attn.forward_with_cache(&xs, cos, sin, attention_mask, true)?;
let xs = self
.self_attn
.forward_with_cache(&xs, cos, sin, attention_mask, true)?;
let xs = (residual
+ xs.affine(
self.scale_depth as f64 / (self.num_hidden_layers as f64).sqrt(),
@@ -220,11 +222,11 @@ impl MiniCPMModel {
})
}
pub fn forward(&mut self, input_ids: &Tensor, position_id: usize) -> Result<Tensor> {
pub fn forward(&mut self, input_ids: &Tensor, position_id: usize) -> Result<Tensor> {
let (bs, seq_len) = input_ids.dims2()?;
let input_embeds = self
.embed_tokens
.forward(&input_ids)?
.forward(input_ids)?
.affine(self.cfg.scale_emb, 0.0)?;
let attention_mask: Option<&Tensor> = {
if seq_len <= 1 {
@@ -238,7 +240,7 @@ impl MiniCPMModel {
)?)
}
};
let (cos, sin) = self.rope_emb.forward(position_id, seq_len)?;
let mut hidden_states = input_embeds;
for decode_layer in &self.layers {
@@ -258,7 +260,7 @@ impl MiniCPMModel {
let (bs, seq_len) = input_ids.dims2()?;
let input_embeds = self
.embed_tokens
.forward(&input_ids)?
.forward(input_ids)?
.affine(self.cfg.scale_emb, 0.0)?;
let attention_mask: Option<&Tensor> = {
if seq_len <= 1 {