添加 rerank 和 embedding模型支持,并且添加 onnx 模型

src/api/mod.rs

@@ -7,7 +7,7 @@ use aha::process::cleanup_pid_file;
 use aha::utils::string_to_static_str;
 use aha_openai_dive::v1::resources::chat::ChatCompletionParameters;
 use rocket::futures::StreamExt;
-use rocket::serde::{Serialize, json::Json};
+use rocket::serde::{Deserialize, Serialize, json::Json};
 use rocket::{
     Request, State,
     futures::Stream,
@@ -16,6 +16,7 @@ use rocket::{
     post,
     response::{Responder, stream::TextStream},
 };
+use serde_json::Value;
 use tokio::sync::RwLock;
 
 // ASR (Automatic Speech Recognition) API module
@@ -191,6 +192,191 @@ pub(crate) async fn speech(req: Json<ChatCompletionParameters>) -> (Status, Stri
     }
 }
 
+#[derive(Debug, Deserialize)]
+pub(crate) struct EmbeddingRequest {
+    pub model: Option<String>,
+    pub input: Value,
+}
+
+#[derive(Debug, Serialize)]
+struct EmbeddingData {
+    object: String,
+    index: usize,
+    embedding: Vec<f32>,
+}
+
+#[derive(Debug, Serialize)]
+struct EmbeddingResponse {
+    object: String,
+    data: Vec<EmbeddingData>,
+    model: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct RerankRequest {
+    pub model: Option<String>,
+    pub query: String,
+    pub documents: Vec<String>,
+    pub top_n: Option<usize>,
+}
+
+#[derive(Debug, Serialize)]
+struct RerankResult {
+    index: usize,
+    relevance_score: f32,
+    document: String,
+}
+
+#[derive(Debug, Serialize)]
+struct RerankResponse {
+    object: String,
+    model: String,
+    results: Vec<RerankResult>,
+}
+
+fn parse_embedding_input(input: &Value) -> anyhow::Result<Vec<String>> {
+    match input {
+        Value::String(s) => Ok(vec![s.clone()]),
+        Value::Array(arr) => {
+            let mut out = Vec::with_capacity(arr.len());
+            for v in arr {
+                let s = v.as_str().ok_or_else(|| {
+                    anyhow::anyhow!("embedding input array must contain only strings")
+                })?;
+                out.push(s.to_string());
+            }
+            if out.is_empty() {
+                return Err(anyhow::anyhow!("embedding input cannot be empty"));
+            }
+            Ok(out)
+        }
+        _ => Err(anyhow::anyhow!(
+            "embedding input must be a string or an array of strings"
+        )),
+    }
+}
+
+fn validate_rerank_input(query: &str, documents: &[String]) -> anyhow::Result<()> {
+    if query.trim().is_empty() {
+        return Err(anyhow::anyhow!("rerank query cannot be empty"));
+    }
+    if documents.is_empty() {
+        return Err(anyhow::anyhow!("rerank documents cannot be empty"));
+    }
+    if documents.iter().any(|doc| doc.trim().is_empty()) {
+        return Err(anyhow::anyhow!(
+            "rerank documents cannot contain empty strings"
+        ));
+    }
+    Ok(())
+}
+
+#[post("/embeddings", data = "<req>")]
+pub(crate) async fn embeddings(req: Json<EmbeddingRequest>) -> (Status, Json<Value>) {
+    let texts = match parse_embedding_input(&req.input) {
+        Ok(v) => v,
+        Err(e) => {
+            return (
+                Status::BadRequest,
+                Json(serde_json::json!({ "error": e.to_string() })),
+            );
+        }
+    };
+    let model_ref = match MODEL.get().cloned() {
+        Some(v) => v,
+        None => {
+            return (
+                Status::ServiceUnavailable,
+                Json(serde_json::json!({ "error": "model not init" })),
+            );
+        }
+    };
+    let mut guard = model_ref.write().await;
+    let embeddings = match guard.instance.embedding(&texts) {
+        Ok(v) => v,
+        Err(e) => {
+            return (
+                Status::BadRequest,
+                Json(serde_json::json!({ "error": e.to_string() })),
+            );
+        }
+    };
+    let model_name = req
+        .model
+        .clone()
+        .unwrap_or_else(|| guard.which_model.openai_model_id().to_string());
+    let data = embeddings
+        .into_iter()
+        .enumerate()
+        .map(|(index, embedding)| EmbeddingData {
+            object: "embedding".to_string(),
+            index,
+            embedding,
+        })
+        .collect::<Vec<_>>();
+    let response = EmbeddingResponse {
+        object: "list".to_string(),
+        data,
+        model: model_name,
+    };
+    (Status::Ok, Json(serde_json::to_value(response).unwrap()))
+}
+
+#[post("/rerank", data = "<req>")]
+pub(crate) async fn rerank(req: Json<RerankRequest>) -> (Status, Json<Value>) {
+    let req = req.into_inner();
+    if let Err(e) = validate_rerank_input(&req.query, &req.documents) {
+        return (
+            Status::BadRequest,
+            Json(serde_json::json!({ "error": e.to_string() })),
+        );
+    }
+
+    let model_ref = match MODEL.get().cloned() {
+        Some(v) => v,
+        None => {
+            return (
+                Status::ServiceUnavailable,
+                Json(serde_json::json!({ "error": "model not init" })),
+            );
+        }
+    };
+
+    let mut guard = model_ref.write().await;
+    let scores = match guard.instance.rerank(&req.query, &req.documents) {
+        Ok(v) => v,
+        Err(e) => {
+            return (
+                Status::BadRequest,
+                Json(serde_json::json!({ "error": e.to_string() })),
+            );
+        }
+    };
+
+    let mut results = scores
+        .into_iter()
+        .enumerate()
+        .map(|(index, relevance_score)| RerankResult {
+            index,
+            relevance_score,
+            document: req.documents[index].clone(),
+        })
+        .collect::<Vec<_>>();
+    results.sort_by(|a, b| b.relevance_score.total_cmp(&a.relevance_score));
+    if let Some(top_n) = req.top_n {
+        results.truncate(top_n.min(results.len()));
+    }
+
+    let response = RerankResponse {
+        object: "list".to_string(),
+        model: req
+            .model
+            .unwrap_or_else(|| guard.which_model.openai_model_id().to_string()),
+        results,
+    };
+    (Status::Ok, Json(serde_json::to_value(response).unwrap()))
+}
+
 // Health check endpoint
 
 #[derive(Serialize)]
@@ -243,63 +429,6 @@ struct ErrorResponse {
     error: String,
 }
 
-/// Convert WhichModel to a display-friendly model ID (kebab-case)
-fn which_model_to_id(which_model: WhichModel) -> &'static str {
-    match which_model {
-        WhichModel::MiniCPM4_0_5B => "minicpm4-0.5b",
-        WhichModel::Qwen2_5vl3B => "qwen2.5vl-3b",
-        WhichModel::Qwen2_5vl7B => "qwen2.5vl-7b",
-        WhichModel::Qwen3_0_6B => "qwen3-0.6b",
-        WhichModel::Qwen3_5_0_8B => "qwen3.5-0.8b",
-        WhichModel::Qwen3_5_2B => "qwen3.5-2b",
-        WhichModel::Qwen3_5_4B => "qwen3.5-4b",
-        WhichModel::Qwen3_5_9B => "qwen3.5-9b",
-        WhichModel::Qwen3_5Gguf => "qwen3.5-gguf",
-        WhichModel::Qwen3ASR0_6B => "qwen3asr-0.6b",
-        WhichModel::Qwen3ASR1_7B => "qwen3asr-1.7b",
-        WhichModel::Qwen3vl2B => "qwen3vl-2b",
-        WhichModel::Qwen3vl4B => "qwen3vl-4b",
-        WhichModel::Qwen3vl8B => "qwen3vl-8b",
-        WhichModel::Qwen3vl32B => "qwen3vl-32b",
-        WhichModel::DeepSeekOCR => "deepseek-ocr",
-        WhichModel::DeepSeekOCR2 => "deepseek-ocr2",
-        WhichModel::HunyuanOCR => "hunyuan-ocr",
-        WhichModel::PaddleOCRVL => "paddleocr-vl",
-        WhichModel::PaddleOCRVL1_5 => "paddleocr-vl1.5",
-        WhichModel::RMBG2_0 => "rmbg2.0",
-        WhichModel::VoxCPM => "voxcpm",
-        WhichModel::VoxCPM1_5 => "voxcpm1.5",
-        WhichModel::GlmASRNano2512 => "glm-asr-nano-2512",
-        WhichModel::FunASRNano2512 => "fun-asr-nano-2512",
-        WhichModel::GlmOCR => "glm-ocr",
-    }
-}
-
-/// Get the owner/organization name for a model
-fn which_model_to_owner(which_model: WhichModel) -> &'static str {
-    match which_model {
-        WhichModel::MiniCPM4_0_5B => "OpenBMB",
-        WhichModel::Qwen2_5vl3B | WhichModel::Qwen2_5vl7B => "Qwen",
-        WhichModel::Qwen3_0_6B | WhichModel::Qwen3ASR0_6B | WhichModel::Qwen3ASR1_7B => "Qwen",
-        WhichModel::Qwen3vl2B
-        | WhichModel::Qwen3vl4B
-        | WhichModel::Qwen3vl8B
-        | WhichModel::Qwen3vl32B
-        | WhichModel::Qwen3_5Gguf => "Qwen",
-        WhichModel::Qwen3_5_0_8B
-        | WhichModel::Qwen3_5_2B
-        | WhichModel::Qwen3_5_4B
-        | WhichModel::Qwen3_5_9B => "Qwen",
-        WhichModel::DeepSeekOCR | WhichModel::DeepSeekOCR2 => "deepseek-ai",
-        WhichModel::HunyuanOCR => "Tencent-Hunyuan",
-        WhichModel::PaddleOCRVL | WhichModel::PaddleOCRVL1_5 => "PaddlePaddle",
-        WhichModel::RMBG2_0 => "AI-ModelScope",
-        WhichModel::VoxCPM | WhichModel::VoxCPM1_5 => "OpenBMB",
-        WhichModel::GlmASRNano2512 | WhichModel::GlmOCR => "ZhipuAI",
-        WhichModel::FunASRNano2512 => "FunAudioLLM",
-    }
-}
-
 #[get("/models")]
 pub(crate) async fn models() -> (Status, (ContentType, Json<serde_json::Value>)) {
     if let Some(model_ref) = MODEL.get() {
@@ -307,10 +436,10 @@ pub(crate) async fn models() -> (Status, (ContentType, Json<serde_json::Value>))
         let which_model = guard.which_model;
 
         let model_obj = ModelObject {
-            id: which_model_to_id(which_model).to_string(),
+            id: which_model.openai_model_id().to_string(),
             object: "model".to_string(),
             created: None, // We don't track creation time
-            owned_by: which_model_to_owner(which_model).to_string(),
+            owned_by: which_model.owner().to_string(),
         };
         drop(guard);
 
@@ -373,17 +502,63 @@ mod tests {
         assert_eq!(error, Some("model not initialized"));
     }
 
+    #[test]
+    fn test_parse_embedding_input_string() {
+        let input = serde_json::json!("hello");
+        let out = parse_embedding_input(&input).unwrap();
+        assert_eq!(out, vec!["hello".to_string()]);
+    }
+
+    #[test]
+    fn test_parse_embedding_input_array() {
+        let input = serde_json::json!(["a", "b"]);
+        let out = parse_embedding_input(&input).unwrap();
+        assert_eq!(out, vec!["a".to_string(), "b".to_string()]);
+    }
+
+    #[test]
+    fn test_validate_rerank_input() {
+        let query = "hello";
+        let docs = vec!["doc1".to_string(), "doc2".to_string()];
+        assert!(validate_rerank_input(query, &docs).is_ok());
+    }
+
+    #[test]
+    fn test_validate_rerank_input_empty_doc() {
+        let query = "hello";
+        let docs = vec!["".to_string()];
+        assert!(validate_rerank_input(query, &docs).is_err());
+    }
+
     // Test model type classification
     #[test]
     fn test_get_model_type_llm() {
         assert_eq!(WhichModel::Qwen3_0_6B.model_type(), "llm");
-        assert_eq!(WhichModel::Qwen3vl2B.model_type(), "llm");
         assert_eq!(WhichModel::MiniCPM4_0_5B.model_type(), "llm");
-        assert_eq!(WhichModel::Qwen2_5vl3B.model_type(), "llm");
-        assert_eq!(WhichModel::Qwen2_5vl7B.model_type(), "llm");
-        assert_eq!(WhichModel::Qwen3vl4B.model_type(), "llm");
-        assert_eq!(WhichModel::Qwen3vl8B.model_type(), "llm");
-        assert_eq!(WhichModel::Qwen3vl32B.model_type(), "llm");
+    }
+
+    #[test]
+    fn test_get_model_type_vlm() {
+        assert_eq!(WhichModel::Qwen3vl2B.model_type(), "vlm");
+        assert_eq!(WhichModel::Qwen2_5vl3B.model_type(), "vlm");
+        assert_eq!(WhichModel::Qwen2_5vl7B.model_type(), "vlm");
+        assert_eq!(WhichModel::Qwen3vl4B.model_type(), "vlm");
+        assert_eq!(WhichModel::Qwen3vl8B.model_type(), "vlm");
+        assert_eq!(WhichModel::Qwen3vl32B.model_type(), "vlm");
+    }
+
+    #[test]
+    fn test_get_model_type_embedding() {
+        assert_eq!(WhichModel::Qwen3Embedding0_6B.model_type(), "embedding");
+        assert_eq!(WhichModel::Qwen3Embedding4B.model_type(), "embedding");
+        assert_eq!(WhichModel::Qwen3Embedding8B.model_type(), "embedding");
+    }
+
+    #[test]
+    fn test_get_model_type_reranker() {
+        assert_eq!(WhichModel::Qwen3Reranker0_6B.model_type(), "reranker");
+        assert_eq!(WhichModel::Qwen3Reranker4B.model_type(), "reranker");
+        assert_eq!(WhichModel::Qwen3Reranker8B.model_type(), "reranker");
     }
 
     #[test]
@@ -412,6 +587,14 @@ mod tests {
     #[test]
     fn test_get_model_id() {
         assert_eq!(WhichModel::Qwen3_0_6B.model_id(), "Qwen/Qwen3-0.6B");
+        assert_eq!(
+            WhichModel::Qwen3Reranker4B.model_id(),
+            "Qwen/Qwen3-Reranker-4B"
+        );
+        assert_eq!(
+            WhichModel::Qwen3Reranker8B.model_id(),
+            "Qwen/Qwen3-Reranker-8B"
+        );
         assert_eq!(
             WhichModel::DeepSeekOCR.model_id(),
             "deepseek-ai/DeepSeek-OCR"
@@ -421,26 +604,30 @@ mod tests {
 
     // Test OpenAI-compatible model ID conversion
     #[test]
-    fn test_which_model_to_id() {
-        assert_eq!(which_model_to_id(WhichModel::Qwen3_0_6B), "qwen3-0.6b");
-        assert_eq!(which_model_to_id(WhichModel::DeepSeekOCR), "deepseek-ocr");
-        assert_eq!(which_model_to_id(WhichModel::VoxCPM1_5), "voxcpm1.5");
+    fn test_openai_model_id() {
+        assert_eq!(WhichModel::Qwen3_0_6B.openai_model_id(), "qwen3-0.6b");
         assert_eq!(
-            which_model_to_id(WhichModel::MiniCPM4_0_5B),
-            "minicpm4-0.5b"
+            WhichModel::Qwen3Reranker4B.openai_model_id(),
+            "qwen3-reranker-4b"
         );
+        assert_eq!(
+            WhichModel::Qwen3Reranker8B.openai_model_id(),
+            "qwen3-reranker-8b"
+        );
+        assert_eq!(WhichModel::DeepSeekOCR.openai_model_id(), "deepseek-ocr");
+        assert_eq!(WhichModel::VoxCPM1_5.openai_model_id(), "voxcpm1.5");
+        assert_eq!(WhichModel::MiniCPM4_0_5B.openai_model_id(), "minicpm4-0.5b");
     }
 
     // Test owner/organization mapping
     #[test]
-    fn test_which_model_to_owner() {
-        assert_eq!(which_model_to_owner(WhichModel::Qwen3_0_6B), "Qwen");
-        assert_eq!(which_model_to_owner(WhichModel::DeepSeekOCR), "deepseek-ai");
-        assert_eq!(which_model_to_owner(WhichModel::VoxCPM1_5), "OpenBMB");
-        assert_eq!(
-            which_model_to_owner(WhichModel::HunyuanOCR),
-            "Tencent-Hunyuan"
-        );
+    fn test_model_owner() {
+        assert_eq!(WhichModel::Qwen3_0_6B.owner(), "Qwen");
+        assert_eq!(WhichModel::Qwen3Reranker4B.owner(), "Qwen");
+        assert_eq!(WhichModel::Qwen3Reranker8B.owner(), "Qwen");
+        assert_eq!(WhichModel::DeepSeekOCR.owner(), "deepseek-ai");
+        assert_eq!(WhichModel::VoxCPM1_5.owner(), "OpenBMB");
+        assert_eq!(WhichModel::HunyuanOCR.owner(), "Tencent-Hunyuan");
     }
 }
 

src/exec/fun_asr_nano.rs

@@ -27,7 +27,9 @@ impl ExecModel for FunASRNanoExec {
         println!("Time elapsed in load model is: {:?}", i_duration);
 
         // Create ChatCompletionParameters for ASR
-        let url = &input[1];
+        let url = input.get(1).ok_or_else(|| {
+            anyhow::anyhow!("fun-asr-nano requires a second input: audio path or URL")
+        })?;
         let input_url = if url.starts_with("http://")
             || url.starts_with("https://")
             || url.starts_with("file://")

src/exec/glm_asr_nano.rs

@@ -28,7 +28,9 @@ impl ExecModel for GlmASRNanoExec {
 
         // Create ChatCompletionParameters for ASR
         // Input should be an audio file path
-        let url = &input[1];
+        let url = input.get(1).ok_or_else(|| {
+            anyhow::anyhow!("glm-asr-nano requires a second input: audio path or URL")
+        })?;
         let input_url = if url.starts_with("http://")
             || url.starts_with("https://")
             || url.starts_with("file://")

src/exec/mod.rs

@@ -14,6 +14,8 @@ pub mod qwen2_5vl;
 pub mod qwen3;
 pub mod qwen3_5;
 pub mod qwen3_asr;
+pub mod qwen3_embedding;
+pub mod qwen3_reranker;
 pub mod qwen3vl;
 pub mod rmbg2_0;
 pub mod voxcpm;

src/exec/qwen2_5vl.rs

@@ -19,7 +19,9 @@ impl ExecModel for Qwen2_5vlExec {
         } else {
             input_text.clone()
         };
-        let url = &input[1];
+        let url = input.get(1).ok_or_else(|| {
+            anyhow::anyhow!("qwen2.5vl requires a second input: image path or URL")
+        })?;
         let input_url = if url.starts_with("http://")
             || url.starts_with("https://")
             || url.starts_with("file://")

src/exec/qwen3_5.rs

@@ -148,7 +148,9 @@ impl ExecModel for Qwen3_5Exec {
         let mut model = Qwen3_5GenerateModel::init(weight_path, None, None)?;
         let i_duration = i_start.elapsed();
         println!("Time elapsed in load model is: {:?}", i_duration);
-        let url = &input[1];
+        let url = input
+            .get(1)
+            .ok_or_else(|| anyhow!("qwen3.5 requires a second input: image/video path or URL"))?;
         let input_url = if url.starts_with("http://")
             || url.starts_with("https://")
             || url.starts_with("file://")

src/exec/qwen3_embedding.rs

@@ -0,0 +1,39 @@
+use std::time::Instant;
+
+use anyhow::Result;
+
+use crate::exec::ExecModel;
+use crate::models::qwen3_embedding::generate::Qwen3EmbeddingModel;
+use crate::utils::get_file_path;
+
+pub struct Qwen3EmbeddingExec;
+
+impl ExecModel for Qwen3EmbeddingExec {
+    fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
+        let input_text = input
+            .first()
+            .ok_or_else(|| anyhow::anyhow!("embedding run requires one text input"))?;
+        let text = if input_text.starts_with("file://") {
+            let path = get_file_path(input_text)?;
+            std::fs::read_to_string(path)?
+        } else {
+            input_text.clone()
+        };
+
+        let i_start = Instant::now();
+        let mut model = Qwen3EmbeddingModel::init(weight_path, None, None)?;
+        println!("Time elapsed in load model is: {:?}", i_start.elapsed());
+
+        let i_start = Instant::now();
+        let embedding = model.embed(&[text])?;
+        println!("Time elapsed in embedding is: {:?}", i_start.elapsed());
+
+        let output_json = serde_json::to_string_pretty(&embedding)?;
+        println!("{}", output_json);
+        if let Some(out) = output {
+            std::fs::write(out, output_json)?;
+            println!("Output saved to: {}", out);
+        }
+        Ok(())
+    }
+}

src/exec/qwen3_reranker.rs

@@ -0,0 +1,87 @@
+use std::time::Instant;
+
+use anyhow::{Result, anyhow};
+use serde::Serialize;
+
+use crate::exec::ExecModel;
+use crate::models::qwen3_reranker::generate::Qwen3RerankerModel;
+
+#[derive(Debug, Serialize)]
+struct RerankItem {
+    index: usize,
+    score: f32,
+    document: String,
+}
+
+pub struct Qwen3RerankerExec;
+
+impl ExecModel for Qwen3RerankerExec {
+    fn run(input: &[String], output: Option<&str>, weight_path: &str) -> Result<()> {
+        if input.len() < 2 {
+            return Err(anyhow!(
+                "reranker run requires two inputs: <query> <documents-source>"
+            ));
+        }
+        let query = input[0].clone();
+        let docs_source = input[1].clone();
+        let documents = parse_documents_source(&docs_source)?;
+        if documents.is_empty() {
+            return Err(anyhow!("documents list is empty"));
+        }
+
+        let mut model = Qwen3RerankerModel::init(weight_path, None, None)?;
+        let i_start = Instant::now();
+        let scores = model.rerank(&query, &documents)?;
+        println!("Time elapsed in rerank is: {:?}", i_start.elapsed());
+
+        let mut ranked = scores
+            .into_iter()
+            .enumerate()
+            .map(|(index, score)| RerankItem {
+                index,
+                score,
+                document: documents[index].clone(),
+            })
+            .collect::<Vec<_>>();
+        ranked.sort_by(|a, b| b.score.total_cmp(&a.score));
+
+        let output_json = serde_json::to_string_pretty(&ranked)?;
+        println!("{}", output_json);
+
+        let output_path = output
+            .map(|o| o.to_string())
+            .unwrap_or_else(|| format!("qwen3-rerank-{}.json", chrono::Utc::now().timestamp()));
+        std::fs::write(&output_path, output_json.as_bytes())?;
+        println!("Generate rerank output to {}", output_path);
+        Ok(())
+    }
+}
+
+fn parse_documents_source(source: &str) -> Result<Vec<String>> {
+    if source.starts_with("file://") {
+        let path = source.trim_start_matches("file://");
+        return read_documents_file(path);
+    }
+    if std::path::Path::new(source).exists() {
+        return read_documents_file(source);
+    }
+
+    let docs = source
+        .split("|||")
+        .map(str::trim)
+        .filter(|x| !x.is_empty())
+        .map(|x| x.to_string())
+        .collect::<Vec<_>>();
+    Ok(docs)
+}
+
+fn read_documents_file(path: &str) -> Result<Vec<String>> {
+    let content = std::fs::read_to_string(path)?;
+    let docs = content
+        .lines()
+        .map(str::trim)
+        .filter(|line| !line.is_empty())
+        .map(|line| line.to_string())
+        .collect::<Vec<_>>();
+    Ok(docs)
+}

src/main.rs

@@ -2,7 +2,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::{net::IpAddr, str::FromStr, sync::Arc};
 
 use aha::{
-    models::WhichModel,
+    models::{LISTED_MODELS, ModelArtifactFormat, WhichModel},
     process::{cleanup_pid_file, create_pid_file},
     utils::{download_model, get_default_save_dir},
 };
@@ -242,37 +242,9 @@ struct ModelInfo {
 
 /// List all supported models
 fn run_list(args: ListArgs) -> anyhow::Result<()> {
-    let models = [
-        WhichModel::MiniCPM4_0_5B,
-        WhichModel::Qwen2_5vl3B,
-        WhichModel::Qwen2_5vl7B,
-        WhichModel::Qwen3_0_6B,
-        WhichModel::Qwen3_5_0_8B,
-        WhichModel::Qwen3_5_2B,
-        WhichModel::Qwen3_5_4B,
-        WhichModel::Qwen3_5_9B,
-        WhichModel::Qwen3ASR0_6B,
-        WhichModel::Qwen3ASR1_7B,
-        WhichModel::Qwen3vl2B,
-        WhichModel::Qwen3vl4B,
-        WhichModel::Qwen3vl8B,
-        WhichModel::Qwen3vl32B,
-        WhichModel::DeepSeekOCR,
-        WhichModel::DeepSeekOCR2,
-        WhichModel::HunyuanOCR,
-        WhichModel::PaddleOCRVL,
-        WhichModel::PaddleOCRVL1_5,
-        WhichModel::RMBG2_0,
-        WhichModel::VoxCPM,
-        WhichModel::VoxCPM1_5,
-        WhichModel::GlmASRNano2512,
-        WhichModel::FunASRNano2512,
-        WhichModel::GlmOCR,
-    ];
-
     if args.json {
         // JSON output
-        let model_infos: Vec<ModelInfo> = models
+        let model_infos: Vec<ModelInfo> = LISTED_MODELS
             .iter()
             .map(|model| {
                 let possible_value = model.to_possible_value().unwrap();
@@ -294,11 +266,11 @@ fn run_list(args: ListArgs) -> anyhow::Result<()> {
             "Model Name", "ModelScope ID", "Download"
         );
         println!("{}", "-".repeat(80));
-        for model in models {
+        for model in LISTED_MODELS {
             let possible_value = model.to_possible_value().unwrap();
             let name = possible_value.get_name();
             let id = model.model_id();
-            let download_status = if is_model_downloaded(model) {
+            let download_status = if is_model_downloaded(*model) {
                 "  ✔"
             } else {
                 ""
@@ -310,6 +282,46 @@ fn run_list(args: ListArgs) -> anyhow::Result<()> {
     Ok(())
 }
 
+async fn resolve_model_paths_for_server(
+    model: WhichModel,
+    weight_path: Option<String>,
+    save_dir: Option<String>,
+    download_retries: Option<u32>,
+    gguf_path: Option<String>,
+    mmproj_path: Option<String>,
+    allow_download: bool,
+) -> anyhow::Result<(String, Option<String>, Option<String>)> {
+    match model.artifact_format() {
+        ModelArtifactFormat::Gguf => {
+            if gguf_path.is_none() {
+                return Err(anyhow!("gguf model path is required"));
+            }
+            Ok(("GGUF".to_string(), gguf_path, mmproj_path))
+        }
+        ModelArtifactFormat::Safetensors => {
+            let model_id = model.model_id();
+            let model_path = match weight_path {
+                Some(path) => path,
+                None if allow_download => {
+                    let save_dir = match save_dir {
+                        Some(dir) => dir,
+                        None => get_default_save_dir().expect("Failed to get home directory"),
+                    };
+                    let max_retries = download_retries.unwrap_or(3);
+                    download_model(model_id, &save_dir, max_retries).await?;
+                    save_dir + "/" + model_id
+                }
+                None => get_default_weight_path(model),
+            };
+            Ok((model_path, None, None))
+        }
+        ModelArtifactFormat::Onnx => Err(anyhow!(
+            "onnx runtime is not integrated yet for model {}",
+            model.openai_model_id()
+        )),
+    }
+}
+
 /// Run the 'cli' subcommand: download model (if needed) and start service
 async fn run_cli(args: CliArgs) -> anyhow::Result<()> {
     let CliArgs {
@@ -320,28 +332,16 @@ async fn run_cli(args: CliArgs) -> anyhow::Result<()> {
         gguf_path,
         mmproj_path,
     } = args;
-    let model_id = common.model.model_id();
-
-    let (model_path, gguf, mmproj) = if model_id.eq("GGUF") {
-        if gguf_path.is_none() {
-            return Err(anyhow!("gguf model path is required"));
-        }
-        ("GGUF".to_string(), gguf_path, mmproj_path)
-    } else {
-        let model_path = match weight_path {
-            Some(path) => path,
-            None => {
-                let save_dir = match save_dir {
-                    Some(dir) => dir,
-                    None => get_default_save_dir().expect("Failed to get home directory"),
-                };
-                let max_retries = download_retries.unwrap_or(3);
-                download_model(model_id, &save_dir, max_retries).await?;
-                save_dir + "/" + model_id
-            }
-        };
-        (model_path, None, None)
-    };
+    let (model_path, gguf, mmproj) = resolve_model_paths_for_server(
+        common.model,
+        weight_path,
+        save_dir,
+        download_retries,
+        gguf_path,
+        mmproj_path,
+        true,
+    )
+    .await?;
 
     init(common.model, model_path, gguf, mmproj)?;
     start_http_server(common.address, common.port, common.allow_remote_shutdown).await?;
@@ -357,19 +357,16 @@ async fn run_serv(args: ServArgs) -> anyhow::Result<()> {
         gguf_path,
         mmproj_path,
     } = args;
-    let model_id = common.model.model_id();
-    let (model_path, gguf, mmproj) = if model_id.eq("GGUF") {
-        if gguf_path.is_none() {
-            return Err(anyhow!("gguf model path is required"));
-        }
-        ("GGUF".to_string(), gguf_path, mmproj_path)
-    } else {
-        let model_path = match weight_path {
-            Some(path) => path,
-            None => get_default_weight_path(common.model),
-        };
-        (model_path, None, None)
-    };
+    let (model_path, gguf, mmproj) = resolve_model_paths_for_server(
+        common.model,
+        weight_path,
+        None,
+        None,
+        gguf_path,
+        mmproj_path,
+        false,
+    )
+    .await?;
 
     init(common.model, model_path, gguf, mmproj)?;
     start_http_server(common.address, common.port, common.allow_remote_shutdown).await?;
@@ -426,6 +423,12 @@ async fn run_download(args: DownloadArgs) -> anyhow::Result<()> {
         download_retries,
     } = args;
     let model_id = model.model_id();
+    if !model.is_download_managed() {
+        return Err(anyhow!(
+            "{} does not use managed model download. Please provide local artifact path directly when serving/running.",
+            model.openai_model_id()
+        ));
+    }
 
     let save_dir = match save_dir {
         Some(dir) => dir,
@@ -473,6 +476,18 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
             use aha::exec::qwen3::Qwen3Exec;
             Qwen3Exec::run(&input, output.as_deref(), &weight_path)?;
         }
+        WhichModel::Qwen3Embedding0_6B
+        | WhichModel::Qwen3Embedding4B
+        | WhichModel::Qwen3Embedding8B => {
+            use aha::exec::qwen3_embedding::Qwen3EmbeddingExec;
+            Qwen3EmbeddingExec::run(&input, output.as_deref(), &weight_path)?;
+        }
+        WhichModel::Qwen3Reranker0_6B
+        | WhichModel::Qwen3Reranker4B
+        | WhichModel::Qwen3Reranker8B => {
+            use aha::exec::qwen3_reranker::Qwen3RerankerExec;
+            Qwen3RerankerExec::run(&input, output.as_deref(), &weight_path)?;
+        }
         WhichModel::Qwen3_5_0_8B => {
             use aha::exec::qwen3_5::Qwen3_5Exec;
             Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
@@ -489,7 +504,19 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> {
             use aha::exec::qwen3_5::Qwen3_5Exec;
             Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
         }
-        WhichModel::Qwen3_5Gguf => {
+        WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2 => {
+            use aha::exec::qwen3_5::Qwen3_5Exec;
+            Qwen3_5Exec::run(&input, output.as_deref(), &weight_path)?;
+        }
+        WhichModel::Qwen3_5Gguf
+        | WhichModel::Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf
+        | WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf
+        | WhichModel::Qwen3_5_0_8BUnslothGguf
+        | WhichModel::Qwen3_5_2BUnslothGguf
+        | WhichModel::Qwen3_5_4BUnslothGguf
+        | WhichModel::Qwen3_5_0_8BLmstudioGguf
+        | WhichModel::Qwen3_5_2BLmstudioGguf
+        | WhichModel::Qwen3_5_4BLmstudioGguf => {
             use aha::exec::qwen3_5::Qwen3_5Exec;
             Qwen3_5Exec::run_gguf(&input, output.as_deref(), gguf_path, mmproj_path)?;
         }
@@ -734,6 +761,12 @@ pub(crate) async fn start_http_server(
     builder = builder.mount("/audio", routes![api::speech, api::transcriptions]);
     // /v1/audio/transcriptions (OpenAI standard ASR transcription endpoint)
     builder = builder.mount("/v1/audio", routes![api::transcriptions]);
+    // /embeddings and /v1/embeddings (OpenAI-compatible embeddings endpoint)
+    builder = builder.mount("/", routes![api::embeddings]);
+    builder = builder.mount("/v1", routes![api::embeddings]);
+    // /rerank and /v1/rerank
+    builder = builder.mount("/", routes![api::rerank]);
+    builder = builder.mount("/v1", routes![api::rerank]);
     // Health check and model info endpoints
     builder = builder.mount("/", routes![api::health, api::models]);
     // Shutdown endpoint

src/models/common/mod.rs

@@ -8,6 +8,7 @@ use candle_nn::{
 };
 
 pub mod gguf;
+pub mod retrieval;
 
 use crate::{
     position_embed::rope::{RoPE, apply_rotary_pos_emb, apply_rotary_pos_emb_roformer},

src/models/common/retrieval.rs

@@ -0,0 +1,41 @@
+use anyhow::{Result, anyhow};
+
+pub trait TextEmbeddingBackend {
+    fn embed_texts(&mut self, input: &[String]) -> Result<Vec<Vec<f32>>>;
+}
+
+pub fn l2_normalize(v: &mut [f32]) {
+    let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt();
+    if norm > 0.0 {
+        for x in v.iter_mut() {
+            *x /= norm;
+        }
+    }
+}
+
+pub fn mean_pool(embeddings: &[Vec<f32>]) -> Result<Vec<f32>> {
+    let first = embeddings
+        .first()
+        .ok_or_else(|| anyhow!("embedding hidden state is empty"))?;
+    let mut pooled = vec![0f32; first.len()];
+    for row in embeddings {
+        if row.len() != first.len() {
+            return Err(anyhow!("inconsistent embedding width in hidden state"));
+        }
+        for (idx, value) in row.iter().enumerate() {
+            pooled[idx] += *value;
+        }
+    }
+    let inv = 1.0f32 / embeddings.len() as f32;
+    for value in &mut pooled {
+        *value *= inv;
+    }
+    Ok(pooled)
+}
+
+pub fn cosine_similarity(lhs: &[f32], rhs: &[f32]) -> Result<f32> {
+    if lhs.len() != rhs.len() {
+        return Err(anyhow!("embedding dimension mismatch"));
+    }
+    Ok(lhs.iter().zip(rhs.iter()).map(|(l, r)| l * r).sum::<f32>())
+}

src/models/mod.rs

@@ -15,6 +15,8 @@ pub mod qwen2_5vl;
 pub mod qwen3;
 pub mod qwen3_5;
 pub mod qwen3_asr;
+pub mod qwen3_embedding;
+pub mod qwen3_reranker;
 pub mod qwen3vl;
 pub mod rmbg2_0;
 pub mod voxcpm;
@@ -33,10 +35,18 @@ use crate::models::{
     hunyuan_ocr::generate::HunyuanOCRGenerateModel, minicpm4::generate::MiniCPMGenerateModel,
     paddleocr_vl::generate::PaddleOCRVLGenerateModel, qwen2_5vl::generate::Qwen2_5VLGenerateModel,
     qwen3::generate::Qwen3GenerateModel, qwen3_5::generate::Qwen3_5GenerateModel,
-    qwen3_asr::generate::Qwen3AsrGenerateModel, qwen3vl::generate::Qwen3VLGenerateModel,
+    qwen3_asr::generate::Qwen3AsrGenerateModel, qwen3_embedding::generate::Qwen3EmbeddingModel,
+    qwen3_reranker::generate::Qwen3RerankerModel, qwen3vl::generate::Qwen3VLGenerateModel,
     rmbg2_0::generate::RMBG2_0Model, voxcpm::generate::VoxCPMGenerate,
 };
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ModelArtifactFormat {
+    Safetensors,
+    Gguf,
+    Onnx,
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
 pub enum WhichModel {
     #[value(name = "minicpm4-0.5b", hide = true)]
@@ -47,6 +57,18 @@ pub enum WhichModel {
     Qwen2_5vl7B,
     #[value(name = "qwen3-0.6b", hide = true)]
     Qwen3_0_6B,
+    #[value(name = "qwen3-embedding-0.6b", hide = true)]
+    Qwen3Embedding0_6B,
+    #[value(name = "qwen3-embedding-4b", hide = true)]
+    Qwen3Embedding4B,
+    #[value(name = "qwen3-embedding-8b", hide = true)]
+    Qwen3Embedding8B,
+    #[value(name = "qwen3-reranker-0.6b", hide = true)]
+    Qwen3Reranker0_6B,
+    #[value(name = "qwen3-reranker-4b", hide = true)]
+    Qwen3Reranker4B,
+    #[value(name = "qwen3-reranker-8b", hide = true)]
+    Qwen3Reranker8B,
     #[value(name = "qwen3.5-0.8b", hide = true)]
     Qwen3_5_0_8B,
     #[value(name = "qwen3.5-2b", hide = true)]
@@ -55,8 +77,35 @@ pub enum WhichModel {
     Qwen3_5_4B,
     #[value(name = "qwen3.5-9b", hide = true)]
     Qwen3_5_9B,
+    #[value(
+        name = "qwen3.5-9b-claude-4.6-opus-reasoning-distilled-v2",
+        hide = true
+    )]
+    Qwen3_5_9BClaude46OpusReasoningDistilledV2,
     #[value(name = "qwen3.5-gguf", hide = true)]
     Qwen3_5Gguf,
+    #[value(
+        name = "qwen3.5-4b-claude-4.6-opus-reasoning-distilled-v2-gguf",
+        hide = true
+    )]
+    Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf,
+    #[value(
+        name = "qwen3.5-9b-claude-4.6-opus-reasoning-distilled-v2-gguf",
+        hide = true
+    )]
+    Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf,
+    #[value(name = "qwen3.5-0.8b-unsloth-gguf", hide = true)]
+    Qwen3_5_0_8BUnslothGguf,
+    #[value(name = "qwen3.5-2b-unsloth-gguf", hide = true)]
+    Qwen3_5_2BUnslothGguf,
+    #[value(name = "qwen3.5-4b-unsloth-gguf", hide = true)]
+    Qwen3_5_4BUnslothGguf,
+    #[value(name = "qwen3.5-0.8b-lmstudio-gguf", hide = true)]
+    Qwen3_5_0_8BLmstudioGguf,
+    #[value(name = "qwen3.5-2b-lmstudio-gguf", hide = true)]
+    Qwen3_5_2BLmstudioGguf,
+    #[value(name = "qwen3.5-4b-lmstudio-gguf", hide = true)]
+    Qwen3_5_4BLmstudioGguf,
     #[value(name = "qwen3asr-0.6b", hide = true)]
     Qwen3ASR0_6B,
     #[value(name = "qwen3asr-1.7b", hide = true)]
@@ -93,7 +142,165 @@ pub enum WhichModel {
     GlmOCR,
 }
 
+pub const LISTED_MODELS: &[WhichModel] = &[
+    WhichModel::MiniCPM4_0_5B,
+    WhichModel::Qwen2_5vl3B,
+    WhichModel::Qwen2_5vl7B,
+    WhichModel::Qwen3_0_6B,
+    WhichModel::Qwen3Embedding0_6B,
+    WhichModel::Qwen3Embedding4B,
+    WhichModel::Qwen3Embedding8B,
+    WhichModel::Qwen3Reranker0_6B,
+    WhichModel::Qwen3Reranker4B,
+    WhichModel::Qwen3Reranker8B,
+    WhichModel::Qwen3_5_0_8B,
+    WhichModel::Qwen3_5_2B,
+    WhichModel::Qwen3_5_4B,
+    WhichModel::Qwen3_5_9B,
+    WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2,
+    WhichModel::Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf,
+    WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf,
+    WhichModel::Qwen3_5_0_8BUnslothGguf,
+    WhichModel::Qwen3_5_2BUnslothGguf,
+    WhichModel::Qwen3_5_4BUnslothGguf,
+    WhichModel::Qwen3_5_0_8BLmstudioGguf,
+    WhichModel::Qwen3_5_2BLmstudioGguf,
+    WhichModel::Qwen3_5_4BLmstudioGguf,
+    WhichModel::Qwen3ASR0_6B,
+    WhichModel::Qwen3ASR1_7B,
+    WhichModel::Qwen3vl2B,
+    WhichModel::Qwen3vl4B,
+    WhichModel::Qwen3vl8B,
+    WhichModel::Qwen3vl32B,
+    WhichModel::DeepSeekOCR,
+    WhichModel::DeepSeekOCR2,
+    WhichModel::HunyuanOCR,
+    WhichModel::PaddleOCRVL,
+    WhichModel::PaddleOCRVL1_5,
+    WhichModel::RMBG2_0,
+    WhichModel::VoxCPM,
+    WhichModel::VoxCPM1_5,
+    WhichModel::GlmASRNano2512,
+    WhichModel::FunASRNano2512,
+    WhichModel::GlmOCR,
+];
+
 impl WhichModel {
+    pub fn artifact_format(self) -> ModelArtifactFormat {
+        match self {
+            WhichModel::Qwen3_5Gguf
+            | WhichModel::Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf
+            | WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf
+            | WhichModel::Qwen3_5_0_8BUnslothGguf
+            | WhichModel::Qwen3_5_2BUnslothGguf
+            | WhichModel::Qwen3_5_4BUnslothGguf
+            | WhichModel::Qwen3_5_0_8BLmstudioGguf
+            | WhichModel::Qwen3_5_2BLmstudioGguf
+            | WhichModel::Qwen3_5_4BLmstudioGguf => ModelArtifactFormat::Gguf,
+            _ => ModelArtifactFormat::Safetensors,
+        }
+    }
+
+    pub fn is_download_managed(self) -> bool {
+        !matches!(
+            self.artifact_format(),
+            ModelArtifactFormat::Gguf | ModelArtifactFormat::Onnx
+        )
+    }
+
+    pub fn openai_model_id(self) -> &'static str {
+        match self {
+            WhichModel::MiniCPM4_0_5B => "minicpm4-0.5b",
+            WhichModel::Qwen2_5vl3B => "qwen2.5vl-3b",
+            WhichModel::Qwen2_5vl7B => "qwen2.5vl-7b",
+            WhichModel::Qwen3_0_6B => "qwen3-0.6b",
+            WhichModel::Qwen3Embedding0_6B => "qwen3-embedding-0.6b",
+            WhichModel::Qwen3Embedding4B => "qwen3-embedding-4b",
+            WhichModel::Qwen3Embedding8B => "qwen3-embedding-8b",
+            WhichModel::Qwen3Reranker0_6B => "qwen3-reranker-0.6b",
+            WhichModel::Qwen3Reranker4B => "qwen3-reranker-4b",
+            WhichModel::Qwen3Reranker8B => "qwen3-reranker-8b",
+            WhichModel::Qwen3_5_0_8B => "qwen3.5-0.8b",
+            WhichModel::Qwen3_5_2B => "qwen3.5-2b",
+            WhichModel::Qwen3_5_4B => "qwen3.5-4b",
+            WhichModel::Qwen3_5_9B => "qwen3.5-9b",
+            WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2 => {
+                "qwen3.5-9b-claude-4.6-opus-reasoning-distilled-v2"
+            }
+            WhichModel::Qwen3_5Gguf => "qwen3.5-gguf",
+            WhichModel::Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf => {
+                "qwen3.5-4b-claude-4.6-opus-reasoning-distilled-v2-gguf"
+            }
+            WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf => {
+                "qwen3.5-9b-claude-4.6-opus-reasoning-distilled-v2-gguf"
+            }
+            WhichModel::Qwen3_5_0_8BUnslothGguf => "qwen3.5-0.8b-unsloth-gguf",
+            WhichModel::Qwen3_5_2BUnslothGguf => "qwen3.5-2b-unsloth-gguf",
+            WhichModel::Qwen3_5_4BUnslothGguf => "qwen3.5-4b-unsloth-gguf",
+            WhichModel::Qwen3_5_0_8BLmstudioGguf => "qwen3.5-0.8b-lmstudio-gguf",
+            WhichModel::Qwen3_5_2BLmstudioGguf => "qwen3.5-2b-lmstudio-gguf",
+            WhichModel::Qwen3_5_4BLmstudioGguf => "qwen3.5-4b-lmstudio-gguf",
+            WhichModel::Qwen3ASR0_6B => "qwen3asr-0.6b",
+            WhichModel::Qwen3ASR1_7B => "qwen3asr-1.7b",
+            WhichModel::Qwen3vl2B => "qwen3vl-2b",
+            WhichModel::Qwen3vl4B => "qwen3vl-4b",
+            WhichModel::Qwen3vl8B => "qwen3vl-8b",
+            WhichModel::Qwen3vl32B => "qwen3vl-32b",
+            WhichModel::DeepSeekOCR => "deepseek-ocr",
+            WhichModel::DeepSeekOCR2 => "deepseek-ocr2",
+            WhichModel::HunyuanOCR => "hunyuan-ocr",
+            WhichModel::PaddleOCRVL => "paddleocr-vl",
+            WhichModel::PaddleOCRVL1_5 => "paddleocr-vl1.5",
+            WhichModel::RMBG2_0 => "rmbg2.0",
+            WhichModel::VoxCPM => "voxcpm",
+            WhichModel::VoxCPM1_5 => "voxcpm1.5",
+            WhichModel::GlmASRNano2512 => "glm-asr-nano-2512",
+            WhichModel::FunASRNano2512 => "fun-asr-nano-2512",
+            WhichModel::GlmOCR => "glm-ocr",
+        }
+    }
+
+    pub fn owner(self) -> &'static str {
+        match self {
+            WhichModel::MiniCPM4_0_5B => "OpenBMB",
+            WhichModel::Qwen2_5vl3B | WhichModel::Qwen2_5vl7B => "Qwen",
+            WhichModel::Qwen3_0_6B
+            | WhichModel::Qwen3Embedding0_6B
+            | WhichModel::Qwen3Embedding4B
+            | WhichModel::Qwen3Embedding8B
+            | WhichModel::Qwen3Reranker0_6B
+            | WhichModel::Qwen3Reranker4B
+            | WhichModel::Qwen3Reranker8B
+            | WhichModel::Qwen3ASR0_6B
+            | WhichModel::Qwen3ASR1_7B => "Qwen",
+            WhichModel::Qwen3vl2B
+            | WhichModel::Qwen3vl4B
+            | WhichModel::Qwen3vl8B
+            | WhichModel::Qwen3vl32B
+            | WhichModel::Qwen3_5Gguf => "Qwen",
+            WhichModel::Qwen3_5_0_8B
+            | WhichModel::Qwen3_5_2B
+            | WhichModel::Qwen3_5_4B
+            | WhichModel::Qwen3_5_9B => "Qwen",
+            WhichModel::Qwen3_5_0_8BUnslothGguf
+            | WhichModel::Qwen3_5_2BUnslothGguf
+            | WhichModel::Qwen3_5_4BUnslothGguf => "unsloth",
+            WhichModel::Qwen3_5_0_8BLmstudioGguf
+            | WhichModel::Qwen3_5_2BLmstudioGguf
+            | WhichModel::Qwen3_5_4BLmstudioGguf => "lmstudio-community",
+            WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2
+            | WhichModel::Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf
+            | WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf => "Jackrong",
+            WhichModel::DeepSeekOCR | WhichModel::DeepSeekOCR2 => "deepseek-ai",
+            WhichModel::HunyuanOCR => "Tencent-Hunyuan",
+            WhichModel::PaddleOCRVL | WhichModel::PaddleOCRVL1_5 => "PaddlePaddle",
+            WhichModel::RMBG2_0 => "AI-ModelScope",
+            WhichModel::VoxCPM | WhichModel::VoxCPM1_5 => "OpenBMB",
+            WhichModel::GlmASRNano2512 | WhichModel::GlmOCR => "ZhipuAI",
+            WhichModel::FunASRNano2512 => "FunAudioLLM",
+        }
+    }
+
     /// Get the ModelScope model ID for this model variant
     pub fn model_id(self) -> &'static str {
         match self {
@@ -101,11 +308,32 @@ impl WhichModel {
             WhichModel::Qwen2_5vl3B => "Qwen/Qwen2.5-VL-3B-Instruct",
             WhichModel::Qwen2_5vl7B => "Qwen/Qwen2.5-VL-7B-Instruct",
             WhichModel::Qwen3_0_6B => "Qwen/Qwen3-0.6B",
+            WhichModel::Qwen3Embedding0_6B => "Qwen/Qwen3-Embedding-0.6B",
+            WhichModel::Qwen3Embedding4B => "Qwen/Qwen3-Embedding-4B",
+            WhichModel::Qwen3Embedding8B => "Qwen/Qwen3-Embedding-8B",
+            WhichModel::Qwen3Reranker0_6B => "Qwen/Qwen3-Reranker-0.6B",
+            WhichModel::Qwen3Reranker4B => "Qwen/Qwen3-Reranker-4B",
+            WhichModel::Qwen3Reranker8B => "Qwen/Qwen3-Reranker-8B",
             WhichModel::Qwen3_5_0_8B => "Qwen/Qwen3.5-0.8B",
             WhichModel::Qwen3_5_2B => "Qwen/Qwen3.5-2B",
             WhichModel::Qwen3_5_4B => "Qwen/Qwen3.5-4B",
             WhichModel::Qwen3_5_9B => "Qwen/Qwen3.5-9B",
+            WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2 => {
+                "Jackrong/Qwen3.5-9B-Claude-4.6-Opus-Reasoning-Distilled-v2"
+            }
             WhichModel::Qwen3_5Gguf => "GGUF",
+            WhichModel::Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf => {
+                "Jackrong/Qwen3.5-4B-Claude-4.6-Opus-Reasoning-Distilled-v2-GGUF"
+            }
+            WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf => {
+                "Jackrong/Qwen3.5-9B-Claude-4.6-Opus-Reasoning-Distilled-v2-GGUF"
+            }
+            WhichModel::Qwen3_5_0_8BUnslothGguf => "unsloth/Qwen3.5-0.8B-GGUF",
+            WhichModel::Qwen3_5_2BUnslothGguf => "unsloth/Qwen3.5-2B-GGUF",
+            WhichModel::Qwen3_5_4BUnslothGguf => "unsloth/Qwen3.5-4B-GGUF",
+            WhichModel::Qwen3_5_0_8BLmstudioGguf => "lmstudio-community/Qwen3.5-0.8B-GGUF",
+            WhichModel::Qwen3_5_2BLmstudioGguf => "lmstudio-community/Qwen3.5-2B-GGUF",
+            WhichModel::Qwen3_5_4BLmstudioGguf => "lmstudio-community/Qwen3.5-4B-GGUF",
             WhichModel::Qwen3ASR0_6B => "Qwen/Qwen3-ASR-0.6B",
             WhichModel::Qwen3ASR1_7B => "Qwen/Qwen3-ASR-1.7B",
             WhichModel::Qwen3vl2B => "Qwen/Qwen3-VL-2B-Instruct",
@@ -131,6 +359,12 @@ impl WhichModel {
         match self {
             // LLM models
             WhichModel::MiniCPM4_0_5B | WhichModel::Qwen3_0_6B => "llm",
+            WhichModel::Qwen3Embedding0_6B
+            | WhichModel::Qwen3Embedding4B
+            | WhichModel::Qwen3Embedding8B => "embedding",
+            WhichModel::Qwen3Reranker0_6B
+            | WhichModel::Qwen3Reranker4B
+            | WhichModel::Qwen3Reranker8B => "reranker",
             WhichModel::Qwen2_5vl3B
             | WhichModel::Qwen2_5vl7B
             | WhichModel::Qwen3vl2B
@@ -141,7 +375,16 @@ impl WhichModel {
             | WhichModel::Qwen3_5_2B
             | WhichModel::Qwen3_5_4B
             | WhichModel::Qwen3_5_9B
+            | WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2
             | WhichModel::Qwen3_5Gguf => "vlm",
+            WhichModel::Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf
+            | WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf
+            | WhichModel::Qwen3_5_0_8BUnslothGguf
+            | WhichModel::Qwen3_5_2BUnslothGguf
+            | WhichModel::Qwen3_5_4BUnslothGguf
+            | WhichModel::Qwen3_5_0_8BLmstudioGguf
+            | WhichModel::Qwen3_5_2BLmstudioGguf
+            | WhichModel::Qwen3_5_4BLmstudioGguf => "vlm",
             // OCR models
             WhichModel::DeepSeekOCR
             | WhichModel::DeepSeekOCR2
@@ -179,6 +422,8 @@ pub enum ModelInstance<'a> {
     MiniCPM4(MiniCPMGenerateModel<'a>),
     Qwen2_5VL(Qwen2_5VLGenerateModel<'a>),
     Qwen3(Qwen3GenerateModel<'a>),
+    Qwen3Embedding(Qwen3EmbeddingModel),
+    Qwen3Reranker(Qwen3RerankerModel),
     Qwen3_5(Qwen3_5GenerateModel<'a>),
     Qwen3ASR(Qwen3AsrGenerateModel<'a>),
     Qwen3VL(Box<Qwen3VLGenerateModel<'a>>),
@@ -198,6 +443,12 @@ impl<'a> GenerateModel for ModelInstance<'a> {
             ModelInstance::MiniCPM4(model) => model.generate(mes),
             ModelInstance::Qwen2_5VL(model) => model.generate(mes),
             ModelInstance::Qwen3(model) => model.generate(mes),
+            ModelInstance::Qwen3Embedding(_) => {
+                Err(anyhow!("embedding model does not support chat completions"))
+            }
+            ModelInstance::Qwen3Reranker(_) => {
+                Err(anyhow!("reranker model does not support chat completions"))
+            }
             ModelInstance::Qwen3_5(model) => model.generate(mes),
             ModelInstance::Qwen3ASR(model) => model.generate(mes),
             ModelInstance::Qwen3VL(model) => model.generate(mes),
@@ -227,6 +478,12 @@ impl<'a> GenerateModel for ModelInstance<'a> {
             ModelInstance::MiniCPM4(model) => model.generate_stream(mes),
             ModelInstance::Qwen2_5VL(model) => model.generate_stream(mes),
             ModelInstance::Qwen3(model) => model.generate_stream(mes),
+            ModelInstance::Qwen3Embedding(_) => Err(anyhow!(
+                "embedding model does not support streaming chat completions"
+            )),
+            ModelInstance::Qwen3Reranker(_) => Err(anyhow!(
+                "reranker model does not support streaming chat completions"
+            )),
             ModelInstance::Qwen3_5(model) => model.generate_stream(mes),
             ModelInstance::Qwen3VL(model) => model.generate_stream(mes),
             ModelInstance::Qwen3ASR(model) => model.generate_stream(mes),
@@ -242,6 +499,22 @@ impl<'a> GenerateModel for ModelInstance<'a> {
     }
 }
 
+impl<'a> ModelInstance<'a> {
+    pub fn embedding(&mut self, input: &[String]) -> Result<Vec<Vec<f32>>> {
+        match self {
+            ModelInstance::Qwen3Embedding(model) => model.embed(input),
+            _ => Err(anyhow!("current model does not support embeddings")),
+        }
+    }
+
+    pub fn rerank(&mut self, query: &str, documents: &[String]) -> Result<Vec<f32>> {
+        match self {
+            ModelInstance::Qwen3Reranker(model) => model.rerank(query, documents),
+            _ => Err(anyhow!("current model does not support reranking")),
+        }
+    }
+}
+
 pub fn load_model<'a>(
     model_type: WhichModel,
     path: &str,
@@ -265,6 +538,18 @@ pub fn load_model<'a>(
             let model = Qwen3GenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3(model)
         }
+        WhichModel::Qwen3Embedding0_6B
+        | WhichModel::Qwen3Embedding4B
+        | WhichModel::Qwen3Embedding8B => {
+            let model = Qwen3EmbeddingModel::init(path, None, None)?;
+            ModelInstance::Qwen3Embedding(model)
+        }
+        WhichModel::Qwen3Reranker0_6B
+        | WhichModel::Qwen3Reranker4B
+        | WhichModel::Qwen3Reranker8B => {
+            let model = Qwen3RerankerModel::init(path, None, None)?;
+            ModelInstance::Qwen3Reranker(model)
+        }
         WhichModel::Qwen3_5_0_8B => {
             let model = Qwen3_5GenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3_5(model)
@@ -281,7 +566,19 @@ pub fn load_model<'a>(
             let model = Qwen3_5GenerateModel::init(path, None, None)?;
             ModelInstance::Qwen3_5(model)
         }
-        WhichModel::Qwen3_5Gguf => {
+        WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2 => {
+            let model = Qwen3_5GenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3_5(model)
+        }
+        WhichModel::Qwen3_5Gguf
+        | WhichModel::Qwen3_5_4BClaude46OpusReasoningDistilledV2Gguf
+        | WhichModel::Qwen3_5_9BClaude46OpusReasoningDistilledV2Gguf
+        | WhichModel::Qwen3_5_0_8BUnslothGguf
+        | WhichModel::Qwen3_5_2BUnslothGguf
+        | WhichModel::Qwen3_5_4BUnslothGguf
+        | WhichModel::Qwen3_5_0_8BLmstudioGguf
+        | WhichModel::Qwen3_5_2BLmstudioGguf
+        | WhichModel::Qwen3_5_4BLmstudioGguf => {
             if gguf.is_none() {
                 return Err(anyhow!("Qwen3_5Gguf gguf model path is required"));
             }

src/models/qwen3/model.rs

@@ -205,7 +205,11 @@ pub struct Qwen3Model {
 
 impl Qwen3Model {
     pub fn new(config: &Qwen3Config, vb: VarBuilder) -> Result<Self> {
-        let vb = vb.pp("model");
+        let vb = if vb.contains_tensor("model.embed_tokens.weight") {
+            vb.pp("model")
+        } else {
+            vb
+        };
         let vocab_size = config.vocab_size;
         let embed_tokens = embedding(vocab_size, config.hidden_size, vb.pp("embed_tokens"))?;
         let mut layers = vec![];
@@ -235,6 +239,19 @@ impl Qwen3Model {
         input_ids: Option<&Tensor>,
         inputs_embeds: Option<&Tensor>,
         seqlen_offset: usize,
+    ) -> Result<Tensor> {
+        let hidden_states = self.forward_hidden(input_ids, inputs_embeds, seqlen_offset)?;
+        let seq_len = hidden_states.dim(1)?;
+        let hidden_state = hidden_states.narrow(1, seq_len - 1, 1)?;
+        let logits = self.lm_head.forward(&hidden_state)?;
+        Ok(logits)
+    }
+
+    pub fn forward_hidden(
+        &mut self,
+        input_ids: Option<&Tensor>,
+        inputs_embeds: Option<&Tensor>,
+        seqlen_offset: usize,
     ) -> Result<Tensor> {
         if input_ids.is_none() && inputs_embeds.is_none() {
             return Err(anyhow::anyhow!(
@@ -271,9 +288,7 @@ impl Qwen3Model {
                 decode_layer.forward(&hidden_states, &cos, &sin, attention_mask.as_ref())?;
         }
         hidden_states = self.norm.forward(&hidden_states)?;
-        let hidden_state = hidden_states.narrow(1, seq_len - 1, 1)?;
-        let logits = self.lm_head.forward(&hidden_state)?;
-        Ok(logits)
+        Ok(hidden_states)
     }
     pub fn embedding_token_id(&self, input_ids: &Tensor) -> Result<Tensor> {
         Ok(self.embed_tokens.forward(input_ids)?)

src/models/qwen3_embedding/config.rs

@@ -0,0 +1,27 @@
+use anyhow::Result;
+
+use crate::models::qwen3::config::Qwen3Config;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Qwen3EmbeddingPoolingStrategy {
+    Mean,
+}
+
+#[derive(Debug, Clone)]
+pub struct Qwen3EmbeddingConfig {
+    pub base: Qwen3Config,
+    pub pooling: Qwen3EmbeddingPoolingStrategy,
+    pub normalize: bool,
+}
+
+impl Qwen3EmbeddingConfig {
+    pub fn load(path: &str) -> Result<Self> {
+        let config_path = format!("{path}/config.json");
+        let base: Qwen3Config = serde_json::from_slice(&std::fs::read(config_path)?)?;
+        Ok(Self {
+            base,
+            pooling: Qwen3EmbeddingPoolingStrategy::Mean,
+            normalize: true,
+        })
+    }
+}

src/models/qwen3_embedding/generate.rs

@@ -0,0 +1,27 @@
+use anyhow::Result;
+use candle_core::{DType, Device};
+
+use crate::models::{
+    common::retrieval::TextEmbeddingBackend, qwen3_embedding::model::Qwen3EmbeddingBackend,
+};
+
+pub struct Qwen3EmbeddingModel {
+    backend: Qwen3EmbeddingBackend,
+}
+
+impl Qwen3EmbeddingModel {
+    pub fn init(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self> {
+        let backend = Qwen3EmbeddingBackend::load(path, device, dtype)?;
+        Ok(Self { backend })
+    }
+
+    pub fn embed(&mut self, input: &[String]) -> Result<Vec<Vec<f32>>> {
+        self.backend.embed_texts(input)
+    }
+}
+
+impl TextEmbeddingBackend for Qwen3EmbeddingModel {
+    fn embed_texts(&mut self, input: &[String]) -> Result<Vec<Vec<f32>>> {
+        self.backend.embed_texts(input)
+    }
+}

src/models/qwen3_embedding/mod.rs

@@ -0,0 +1,3 @@
+pub mod config;
+pub mod generate;
+pub mod model;

src/models/qwen3_embedding/model.rs

@@ -0,0 +1,69 @@
+use anyhow::{Result, anyhow};
+use candle_core::{DType, Device};
+use candle_nn::VarBuilder;
+
+use crate::{
+    models::{
+        common::retrieval::{l2_normalize, mean_pool},
+        qwen3::model::Qwen3Model,
+        qwen3_embedding::config::{Qwen3EmbeddingConfig, Qwen3EmbeddingPoolingStrategy},
+    },
+    tokenizer::TokenizerModel,
+    utils::{find_type_files, get_device, get_dtype},
+};
+
+pub struct Qwen3EmbeddingBackend {
+    tokenizer: TokenizerModel,
+    model: Qwen3Model,
+    device: Device,
+    pooling: Qwen3EmbeddingPoolingStrategy,
+    normalize: bool,
+}
+
+impl Qwen3EmbeddingBackend {
+    pub fn load(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self> {
+        let tokenizer = TokenizerModel::init(path)?;
+        let cfg = Qwen3EmbeddingConfig::load(path)?;
+        let device = get_device(device);
+        let dtype = get_dtype(dtype, cfg.base.torch_dtype.as_str());
+        let model_list = find_type_files(path, "safetensors")?;
+        let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, &device)? };
+        let model = Qwen3Model::new(&cfg.base, vb)?;
+        Ok(Self {
+            tokenizer,
+            model,
+            device,
+            pooling: cfg.pooling,
+            normalize: cfg.normalize,
+        })
+    }
+
+    pub fn embed_texts(&mut self, input: &[String]) -> Result<Vec<Vec<f32>>> {
+        if input.is_empty() {
+            return Err(anyhow!("embedding input cannot be empty"));
+        }
+        let mut out = Vec::with_capacity(input.len());
+        for text in input {
+            out.push(self.embed_one(text)?);
+            self.model.clear_kv_cache();
+        }
+        Ok(out)
+    }
+
+    fn embed_one(&mut self, text: &str) -> Result<Vec<f32>> {
+        let input_ids = self.tokenizer.text_encode(text.to_string(), &self.device)?;
+        let hidden = self
+            .model
+            .forward_hidden(Some(&input_ids), None, 0)?
+            .squeeze(0)?
+            .to_dtype(DType::F32)?;
+        let hidden_vec = hidden.to_vec2::<f32>()?;
+        let mut pooled = match self.pooling {
+            Qwen3EmbeddingPoolingStrategy::Mean => mean_pool(&hidden_vec)?,
+        };
+        if self.normalize {
+            l2_normalize(&mut pooled);
+        }
+        Ok(pooled)
+    }
+}

src/models/qwen3_reranker/config.rs

@@ -0,0 +1,17 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Qwen3RerankerSimilarity {
+    Cosine,
+}
+
+#[derive(Debug, Clone)]
+pub struct Qwen3RerankerConfig {
+    pub similarity: Qwen3RerankerSimilarity,
+}
+
+impl Default for Qwen3RerankerConfig {
+    fn default() -> Self {
+        Self {
+            similarity: Qwen3RerankerSimilarity::Cosine,
+        }
+    }
+}

src/models/qwen3_reranker/generate.rs

@@ -0,0 +1,19 @@
+use anyhow::Result;
+use candle_core::{DType, Device};
+
+use crate::models::qwen3_reranker::model::Qwen3RerankerBackend;
+
+pub struct Qwen3RerankerModel {
+    backend: Qwen3RerankerBackend,
+}
+
+impl Qwen3RerankerModel {
+    pub fn init(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self> {
+        let backend = Qwen3RerankerBackend::load(path, device, dtype)?;
+        Ok(Self { backend })
+    }
+
+    pub fn rerank(&mut self, query: &str, documents: &[String]) -> Result<Vec<f32>> {
+        self.backend.rerank(query, documents)
+    }
+}

src/models/qwen3_reranker/mod.rs

@@ -0,0 +1,3 @@
+pub mod config;
+pub mod generate;
+pub mod model;

src/models/qwen3_reranker/model.rs

@@ -0,0 +1,51 @@
+use anyhow::{Result, anyhow};
+use candle_core::{DType, Device};
+
+use crate::models::{
+    common::retrieval::cosine_similarity,
+    qwen3_embedding::generate::Qwen3EmbeddingModel,
+    qwen3_reranker::config::{Qwen3RerankerConfig, Qwen3RerankerSimilarity},
+};
+
+pub struct Qwen3RerankerBackend {
+    config: Qwen3RerankerConfig,
+    embedding_backend: Qwen3EmbeddingModel,
+}
+
+impl Qwen3RerankerBackend {
+    pub fn load(path: &str, device: Option<&Device>, dtype: Option<DType>) -> Result<Self> {
+        let embedding_backend = Qwen3EmbeddingModel::init(path, device, dtype)?;
+        Ok(Self {
+            config: Qwen3RerankerConfig::default(),
+            embedding_backend,
+        })
+    }
+
+    pub fn rerank(&mut self, query: &str, documents: &[String]) -> Result<Vec<f32>> {
+        if query.trim().is_empty() {
+            return Err(anyhow!("reranker query cannot be empty"));
+        }
+        if documents.is_empty() {
+            return Err(anyhow!("reranker documents cannot be empty"));
+        }
+
+        let mut batch = Vec::with_capacity(documents.len() + 1);
+        batch.push(query.to_string());
+        batch.extend(documents.iter().cloned());
+
+        let embeddings = self.embedding_backend.embed(&batch)?;
+        let query_embedding = embeddings
+            .first()
+            .ok_or_else(|| anyhow!("failed to produce query embedding"))?;
+
+        let mut scores = Vec::with_capacity(documents.len());
+        for doc_embedding in embeddings.iter().skip(1) {
+            scores.push(match self.config.similarity {
+                Qwen3RerankerSimilarity::Cosine => {
+                    cosine_similarity(query_embedding, doc_embedding)?
+                }
+            });
+        }
+        Ok(scores)
+    }
+}