pub use DType and Tensor

src/lib.rs

@@ -6,4 +6,4 @@ pub mod position_embed;
 pub mod tokenizer;
 pub mod utils;
 
-pub use candle_core::Device;
+pub use candle_core::{DType, Device, Tensor};

src/models/qwen3_5/generate.rs

@@ -64,6 +64,42 @@ impl<'a> Qwen3_5GenerateModel<'a> {
         })
     }
 
+    pub fn init_without_visual(
+        path: &str,
+        device: Option<&Device>,
+        dtype: Option<DType>,
+    ) -> Result<Self> {
+        let model_name = std::path::Path::new(path)
+            .file_name()
+            .and_then(|s| s.to_str())
+            .unwrap_or("qwen3.5");
+        let chat_template = ChatTemplate::init(path)?;
+        let tokenizer = TokenizerModel::init(path)?;
+        let config_path = path.to_string() + "/config.json";
+        let cfg: Qwen3_5Config = serde_json::from_slice(&std::fs::read(config_path)?)?;
+        let device = get_device(device);
+        let cfg_dtype = cfg.text_config.dtype.as_str();
+        let dtype = get_dtype(dtype, cfg_dtype);
+        // let pre_processor = Qwen3VLProcessor::new(path, &device, dtype)?;
+        let pre_processor = None;
+        let model_list = find_type_files(path, "safetensors")?;
+        let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, &device)? };
+        let eos_ids = vec![cfg.text_config.eos_token_id];
+        // let qwen3_5 = Qwen3_5Model::new_from_vb(vb, cfg, eos_ids)?;
+        let qwen3_5 = Qwen3_5Model::new_from_vb_without_visual(vb, cfg, eos_ids)?;
+
+        Ok(Self {
+            chat_template,
+            tokenizer,
+            pre_processor,
+            qwen3_5,
+            device,
+            model_name: model_name.to_string(),
+            repeat_penalty: 1.0,
+            repeat_last_n: 64,
+        })
+    }
+
     pub fn init_from_gguf(
         model_file: &str,
         mmproj_file: Option<&str>,

src/models/qwen3_5/model.rs

@@ -1077,6 +1077,38 @@ impl Qwen3_5Model {
         })
     }
 
+    pub fn new_from_vb_without_visual(
+        vb: VarBuilder,
+        config: Qwen3_5Config,
+        eos_ids: Vec<u32>,
+    ) -> Result<Self> {
+        let vb_m = vb.pp("model");
+        // let visual = Qwen3VLVisionModel::new(config.vision_config.clone(), vb_m.pp("visual"))?;
+        let visual = None;
+        let language_model =
+            Qwen3_5TextModel::new_from_vb(vb_m.pp("language_model"), &config.text_config)?;
+        let lm_head = if config.tie_word_embeddings {
+            Linear::new(language_model.embed_tokens.embeddings().clone(), None)
+        } else {
+            linear_no_bias(
+                config.text_config.hidden_size,
+                config.text_config.vocab_size,
+                vb.pp("lm_head"),
+            )?
+        };
+        Ok(Self {
+            spatial_merge_size: config.vision_config.spatial_merge_size,
+            image_token_id: config.image_token_id,
+            video_token_id: config.video_token_id,
+            vision_start_token_id: config.vision_start_token_id,
+            visual,
+            language_model,
+            lm_head: ProjKind::LinearProj(lm_head),
+            rope_deltas: None,
+            stop_token_ids: eos_ids,
+        })
+    }
+
     pub fn new_from_gguf<R: Read + Seek>(
         gguf: &mut Gguf<R>,
         mmproj_gguf: Option<&mut Gguf<R>>,

tests/test_qwen3_5.rs

@@ -5,6 +5,45 @@ use aha::params::chat::ChatCompletionParameters;
 use anyhow::Result;
 use rocket::futures::StreamExt;
 
+#[test]
+fn qwen3_5_generate_no_visual() -> Result<()> {
+    // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_qwen3_5 qwen3_5_generate_no_visual -r -- --nocapture
+
+    let save_dir =
+        aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?;
+    let model_path = format!("{}/Qwen/Qwen3.5-0.8B/", save_dir);
+
+    let message = r#"
+    {
+        "model": "qwen3.5",
+        "messages": [
+            {
+                "role": "user",
+                "content": [         
+                    {
+                        "type": "text", 
+                        "text": "你好啊"
+                    }
+                ]
+            }
+        ]
+    }
+    "#;
+    // "metadata": {"enable_thinking": "true"}
+    let mes: ChatCompletionParameters = serde_json::from_str(message)?;
+    let i_start = Instant::now();
+    let mut qwen3_5 = Qwen3_5GenerateModel::init_without_visual(&model_path, None, None)?;
+    let i_duration = i_start.elapsed();
+    println!("Time elapsed in load model is: {:?}", i_duration);
+
+    let res = qwen3_5.generate(mes)?;
+    println!("generate: \n {:?}", res);
+    if let Some(usage) = &res.usage {
+        println!("usage: \n {:?}", usage);
+    }
+    Ok(())
+}
+
 #[test]
 fn qwen3_5_generate() -> Result<()> {
     // test with cuda: RUST_BACKTRACE=1 cargo test -F cuda --test test_qwen3_5 qwen3_5_generate -r -- --nocapture