from gguf build chat_template and tokenizer

This commit is contained in:
jhqxxx
2026-03-11 17:30:28 +08:00
parent 86141322bb
commit 4d544ab68f
14 changed files with 374 additions and 118 deletions
+6 -6
View File
@@ -12,6 +12,10 @@ pub struct TokenizerModel {
}
impl TokenizerModel {
pub fn new(tokenizer: Tokenizer) -> Self {
Self { tokenizer }
}
pub fn init(path: &str) -> Result<Self> {
let path = path.to_string();
assert!(
@@ -81,6 +85,8 @@ impl TokenizerModel {
}
tokenizer
};
let len = tokenizer.get_vocab_size(true);
println!("len: {}", len);
Ok(Self { tokenizer })
}
@@ -94,12 +100,6 @@ impl TokenizerModel {
Ok(token_id)
}
pub fn text_encode(&self, text: String, device: &Device) -> Result<Tensor> {
// let token_id = self
// .tokenizer
// .encode(text, true)
// .map_err(|e| anyhow!(format!("tokenizer encode error: {}", e)))?
// .get_ids()
// .to_vec();
let token_id = self.text_encode_vec(text, true)?;
let token_tensor = Tensor::from_slice(&token_id, (1, token_id.len()), device)?;
Ok(token_tensor)