delete some note

This commit is contained in:
jhqxxx
2025-11-02 21:34:43 +08:00
parent ff5f8f87e1
commit b42c5c718c
3 changed files with 17 additions and 23 deletions
+15 -21
View File
@@ -328,25 +328,33 @@ impl Qwen3VLVisionModel {
return Err(anyhow!(format!("grid_thw Expected exactly 3 elements"))); return Err(anyhow!(format!("grid_thw Expected exactly 3 elements")));
}; };
split_idx.push((h * w) as usize); split_idx.push((h * w) as usize);
let num_grid_per_side = (self.num_grid_per_side - 1) as f32; let num_grid_per_side_sub_one = (self.num_grid_per_side - 1) as f32;
let h_idxs = linspace(0.0, num_grid_per_side, h as usize, grid_thw.device())?; let h_idxs = linspace(
let w_idxs = linspace(0.0, num_grid_per_side, w as usize, grid_thw.device())?; 0.0,
num_grid_per_side_sub_one,
h as usize,
grid_thw.device(),
)?;
let w_idxs = linspace(
0.0,
num_grid_per_side_sub_one,
w as usize,
grid_thw.device(),
)?;
let h_idxs_floor = h_idxs.to_dtype(candle_core::DType::U32)?; let h_idxs_floor = h_idxs.to_dtype(candle_core::DType::U32)?;
let w_idxs_floor = w_idxs.to_dtype(candle_core::DType::U32)?; let w_idxs_floor = w_idxs.to_dtype(candle_core::DType::U32)?;
let h_idxs_ceil = h_idxs_floor let h_idxs_ceil = h_idxs_floor
.affine(1.0, 1.0)? .affine(1.0, 1.0)?
.clamp(0u32, num_grid_per_side as u32)?; .clamp(0u32, num_grid_per_side_sub_one as u32)?;
let w_idxs_ceil = w_idxs_floor let w_idxs_ceil = w_idxs_floor
.affine(1.0, 1.0)? .affine(1.0, 1.0)?
.clamp(0u32, num_grid_per_side as u32)?; .clamp(0u32, num_grid_per_side_sub_one as u32)?;
let dh = h_idxs let dh = h_idxs
.sub(&h_idxs_floor.to_dtype(h_idxs.dtype())?)? .sub(&h_idxs_floor.to_dtype(h_idxs.dtype())?)?
.unsqueeze(D::Minus1)?; .unsqueeze(D::Minus1)?;
let dw = w_idxs let dw = w_idxs
.sub(&w_idxs_floor.to_dtype(h_idxs.dtype())?)? .sub(&w_idxs_floor.to_dtype(h_idxs.dtype())?)?
.unsqueeze(0)?; .unsqueeze(0)?;
let base_h = h_idxs_floor let base_h = h_idxs_floor
.affine(self.num_grid_per_side as f64, 0.0)? .affine(self.num_grid_per_side as f64, 0.0)?
.unsqueeze(D::Minus1)?; .unsqueeze(D::Minus1)?;
@@ -415,7 +423,6 @@ impl Qwen3VLVisionModel {
let mut patch_pos_embeds_permute = vec![]; let mut patch_pos_embeds_permute = vec![];
let patch_pos_embeds = split_tensor(&patch_pos_embeds, &split_idx, 0)?; let patch_pos_embeds = split_tensor(&patch_pos_embeds, &split_idx, 0)?;
let merge_size = self.spatial_merge_size; let merge_size = self.spatial_merge_size;
// for i in 0..grid_thw.dim(0)? {
for (i, pos_embed) in patch_pos_embeds.iter().enumerate() { for (i, pos_embed) in patch_pos_embeds.iter().enumerate() {
let [t, h, w] = grid_thw.i(i)?.to_vec1::<u32>()?[..] else { let [t, h, w] = grid_thw.i(i)?.to_vec1::<u32>()?[..] else {
return Err(anyhow!(format!("grid_thw Expected exactly 3 elements"))); return Err(anyhow!(format!("grid_thw Expected exactly 3 elements")));
@@ -443,7 +450,6 @@ impl Qwen3VLVisionModel {
pub fn rot_pos_emb(&self, grid_thw: &Tensor) -> Result<Tensor> { pub fn rot_pos_emb(&self, grid_thw: &Tensor) -> Result<Tensor> {
let merge_size = self.spatial_merge_size; let merge_size = self.spatial_merge_size;
let max_hw = grid_thw.i((.., 1..))?.max_all()?.to_scalar::<u32>()?; let max_hw = grid_thw.i((.., 1..))?.max_all()?.to_scalar::<u32>()?;
let freq_table = self let freq_table = self
.rotary_pos_emb .rotary_pos_emb
@@ -752,17 +758,6 @@ impl Qwen3VLTextModel {
}) })
} }
// fn deepstack_process(
// &self,
// xs: &Tensor,
// visual_pos_masks: &Tensor,
// visual_embeds: &Tensor,
// ) -> Result<Tensor> {
// let visual_nonzero_index = nonzero_index(&visual_pos_masks)?;
// let xs = xs.index_add(&visual_nonzero_index, visual_embeds, 0)?;
// Ok(xs)
// }
pub fn forward( pub fn forward(
&mut self, &mut self,
inputs_embeds: &Tensor, inputs_embeds: &Tensor,
@@ -1170,7 +1165,6 @@ impl Qwen3VLModel {
} }
let mut visual_pos_mask = None; let mut visual_pos_mask = None;
let mut deepstack_visual_embeds = None; let mut deepstack_visual_embeds = None;
// if image_mask.is_some() && video_mask.is_some() {
if let Some(image_mask_) = image_mask { if let Some(image_mask_) = image_mask {
if let Some(video_mask_) = video_mask { if let Some(video_mask_) = video_mask {
let image_mask_ = image_mask_.squeeze(0)?; let image_mask_ = image_mask_.squeeze(0)?;
+1 -1
View File
@@ -129,7 +129,7 @@ impl Qwen3VLProcessor {
) -> Result<Tensor> { ) -> Result<Tensor> {
let img_h = img.height(); let img_h = img.height();
let img_w = img.width(); let img_w = img.width();
// h,w resize成 28的倍数 // h,w resize成 32的倍数
let (resize_h, resize_w) = img_smart_resize( let (resize_h, resize_w) = img_smart_resize(
img_h, img_h,
img_w, img_w,