temporary save

src/tokenizer/mod.rs

@@ -1,4 +1,4 @@
-use anyhow::{Result, anyhow};
+use anyhow::{Ok, Result, anyhow};
 use candle_core::{Device, Tensor};
 use tokenizers::Tokenizer;
 
@@ -23,13 +23,23 @@ impl TokenizerModel {
         Ok(Self { tokenizer })
     }
 
-    pub fn text_encode(&self, text: String, device: &Device) -> Result<Tensor> {
+    pub fn text_encode_vec(&self, text: String, add_special_token: bool) -> Result<Vec<u32>> {
         let token_id = self
             .tokenizer
-            .encode(text, true)
+            .encode(text, add_special_token)
             .map_err(|e| anyhow!(format!("tokenizer encode error: {}", e)))?
             .get_ids()
             .to_vec();
+        Ok(token_id)
+    }
+    pub fn text_encode(&self, text: String, device: &Device) -> Result<Tensor> {
+        // let token_id = self
+        //     .tokenizer
+        //     .encode(text, true)
+        //     .map_err(|e| anyhow!(format!("tokenizer encode error: {}", e)))?
+        //     .get_ids()
+        //     .to_vec();
+        let token_id = self.text_encode_vec(text, true)?;
         let token_tensor = Tensor::from_slice(&token_id, (1, token_id.len()), device)?;
         Ok(token_tensor)
     }