add deploy model

src/models/minicpm4/generate.rs

@@ -53,7 +53,11 @@ impl<'a> MiniCPMGenerateModel<'a> {
 
 impl<'a> GenerateModel for MiniCPMGenerateModel<'a> {
     fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
-        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None);
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
         let mes_render = self.chat_template.apply_chat_template(&mes)?;
         let mut input_ids = self.tokenizer.text_encode(mes_render, &self.device)?;
         let mut seq_len = input_ids.dim(1)?;
@@ -80,8 +84,19 @@ impl<'a> GenerateModel for MiniCPMGenerateModel<'a> {
     fn generate_stream(
         &mut self,
         mes: ChatCompletionParameters,
-    ) -> Result<impl Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>> {
-        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None);
+    ) -> Result<
+        Box<
+            dyn Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>
+                + Send
+                + Unpin
+                + '_,
+        >,
+    > {
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
         let mes_render = self.chat_template.apply_chat_template(&mes)?;
         let mut input_ids = self.tokenizer.text_encode(mes_render, &self.device)?;
         let mut seq_len = input_ids.dim(1)?;
@@ -125,6 +140,6 @@ impl<'a> GenerateModel for MiniCPMGenerateModel<'a> {
             }
             self.minicpm.clear_kv_cache();
         };
-        Ok(stream)
+        Ok(Box::new(Box::pin(stream)))
     }
 }

src/models/mod.rs

@@ -10,12 +10,84 @@ use aha_openai_dive::v1::resources::chat::{
 use anyhow::Result;
 use rocket::futures::Stream;
 
+use crate::models::{
+    minicpm4::generate::MiniCPMGenerateModel, qwen2_5vl::generate::Qwen2_5VLGenerateModel,
+    qwen3vl::generate::Qwen3VLGenerateModel,
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
+pub enum WhichModel {
+    #[value(name = "minicpm4-0.5b")]
+    MiniCPM4_0_5B,
+    #[value(name = "qwen2.5vl-3b")]
+    Qwen2_5vl3B,
+    #[value(name = "qwen3vl-2b")]
+    Qwen3vl2B,
+}
+
 pub trait GenerateModel {
     fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse>;
     fn generate_stream(
         &mut self,
         mes: ChatCompletionParameters,
-    ) -> Result<impl Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>>
-    where
-        Self: Sized;
+    ) -> Result<
+        Box<
+            dyn Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>
+                + Send
+                + Unpin
+                + '_,
+        >,
+    >;
+}
+
+pub enum ModelInstance<'a> {
+    MiniCPM4(MiniCPMGenerateModel<'a>),
+    Qwen2_5VL(Qwen2_5VLGenerateModel<'a>),
+    Qwen3VL(Qwen3VLGenerateModel<'a>),
+}
+
+impl<'a> GenerateModel for ModelInstance<'a> {
+    fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
+        match self {
+            ModelInstance::MiniCPM4(model) => model.generate(mes),
+            ModelInstance::Qwen2_5VL(model) => model.generate(mes),
+            ModelInstance::Qwen3VL(model) => model.generate(mes),
+        }
+    }
+
+    fn generate_stream(
+        &mut self,
+        mes: ChatCompletionParameters,
+    ) -> Result<
+        Box<
+            dyn Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>
+                + Send
+                + Unpin
+                + '_,
+        >,
+    > {
+        match self {
+            ModelInstance::MiniCPM4(model) => model.generate_stream(mes),
+            ModelInstance::Qwen2_5VL(model) => model.generate_stream(mes),
+            ModelInstance::Qwen3VL(model) => model.generate_stream(mes),
+        }
+    }
+}
+
+pub fn load_model(model_type: WhichModel, path: &str) -> Result<ModelInstance<'_>> {
+    let model = match model_type {
+        WhichModel::MiniCPM4_0_5B => {
+            let model = MiniCPMGenerateModel::init(path, None, None)?;
+            ModelInstance::MiniCPM4(model)
+        }
+        WhichModel::Qwen2_5vl3B => {
+            let model = Qwen2_5VLGenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen2_5VL(model)
+        }
+        WhichModel::Qwen3vl2B => {
+            let model = Qwen3VLGenerateModel::init(path, None, None)?;
+            ModelInstance::Qwen3VL(model)
+        }
+    };
+    Ok(model)
 }

src/models/qwen2_5vl/generate.rs

@@ -63,7 +63,11 @@ impl<'a> Qwen2_5VLGenerateModel<'a> {
 
 impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
     fn generate(&mut self, mes: ChatCompletionParameters) -> Result<ChatCompletionResponse> {
-        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None);
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
         let mes_render = self.chat_template.apply_chat_template(&mes)?;
         let input = self.pre_processor.process_info(&mes, &mes_render)?;
         let mut input_ids = self
@@ -122,8 +126,19 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
     fn generate_stream(
         &mut self,
         mes: ChatCompletionParameters,
-    ) -> Result<impl Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>> {
-        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None);
+    ) -> Result<
+        Box<
+            dyn Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>
+                + Send
+                + Unpin
+                + '_,
+        >,
+    > {
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed);
         let mes_render = self.chat_template.apply_chat_template(&mes)?;
         let input = self.pre_processor.process_info(&mes, &mes_render)?;
         let mut input_ids = self
@@ -203,6 +218,6 @@ impl<'a> GenerateModel for Qwen2_5VLGenerateModel<'a> {
             }
             self.qwen2_5_vl.clear_kv_cache();
         };
-        Ok(stream)
+        Ok(Box::new(Box::pin(stream)))
     }
 }

src/models/qwen3vl/generate.rs

@@ -76,7 +76,12 @@ impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> {
             Some(top_p) => top_p,
         };
         let top_k = self.generation_config.top_k;
-        let mut logit_processor = get_logit_processor(Some(temperature), Some(top_p), Some(top_k));
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let mut logit_processor =
+            get_logit_processor(Some(temperature), Some(top_p), Some(top_k), seed);
         let mes_render = self.chat_template.apply_chat_template(&mes)?;
         let input = self.pre_processor.process_info(&mes, &mes_render)?;
         let mut input_ids = self
@@ -123,7 +128,14 @@ impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> {
     fn generate_stream(
         &mut self,
         mes: ChatCompletionParameters,
-    ) -> Result<impl Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>> {
+    ) -> Result<
+        Box<
+            dyn Stream<Item = Result<ChatCompletionChunkResponse, anyhow::Error>>
+                + Send
+                + Unpin
+                + '_,
+        >,
+    > {
         let temperature = match mes.temperature {
             None => self.generation_config.temperature,
             Some(tem) => tem,
@@ -133,7 +145,12 @@ impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> {
             Some(top_p) => top_p,
         };
         let top_k = self.generation_config.top_k;
-        let mut logit_processor = get_logit_processor(Some(temperature), Some(top_p), Some(top_k));
+        let seed = match mes.seed {
+            None => 34562u64,
+            Some(s) => s as u64,
+        };
+        let mut logit_processor =
+            get_logit_processor(Some(temperature), Some(top_p), Some(top_k), seed);
         let mes_render = self.chat_template.apply_chat_template(&mes)?;
         let input = self.pre_processor.process_info(&mes, &mes_render)?;
         let mut input_ids = self
@@ -199,6 +216,6 @@ impl<'a> GenerateModel for Qwen3VLGenerateModel<'a> {
             }
             self.qwen3_vl.clear_kv_cache();
         };
-        Ok(stream)
+        Ok(Box::new(Box::pin(stream)))
     }
 }