add LFM2-1.2B, LFM2.5-1.2B-Instruct

2026-03-23 22:07:41 +08:00
parent 77b244e53e
commit 9224593b78
26 changed files with 1001 additions and 40 deletions
@@ -110,6 +110,14 @@ impl TokenizerModel {
            .map_err(|e| anyhow!(format!("tokenizer encode error{}", e)))?;
        Ok(decode)
    }
+
+    pub fn token_decode_with_special(&self, tokens: Vec<u32>) -> Result<String> {
+        let decode = self
+            .tokenizer
+            .decode(&tokens, false)
+            .map_err(|e| anyhow!(format!("tokenizer encode error{}", e)))?;
+        Ok(decode)
+    }
 }

 pub fn sentencepiece_encode(