use anyhow::Result; use candle_core::{D, Tensor}; use candle_nn::{BatchNorm, Conv1d, Conv2d, Module, ModuleT, VarBuilder, ops::sigmoid}; use crate::{ models::common::{get_batch_norm, get_conv1d, get_conv2d}, utils::tensor_utils::{pool1d, statistics_pooling}, }; pub struct Shortcut { conv_0: Conv2d, bn_1: BatchNorm, stride: usize, } impl Shortcut { pub fn new( vb: VarBuilder, in_c: usize, out_c: usize, ks: usize, padding: usize, stride: usize, bias: bool, ) -> Result { let conv_0 = get_conv2d(vb.pp("0"), in_c, out_c, ks, padding, 1, 1, 1, bias)?; let bn_1 = get_batch_norm(vb.pp("1"), 1e-5, out_c, true)?; Ok(Self { conv_0, bn_1, stride }) } pub fn forward(&self, x: &Tensor) -> Result { let mut x = self.conv_0.forward(x)?; if self.stride != 1 { let h_dim = x.dim(2)?; let half_h = h_dim / 2; let indices = Tensor::arange(0u32, half_h as u32, x.device())?.affine(2.0, 0.0)?; x = x.index_select(&indices, 2)?; } x = self.bn_1.forward_t(&x, false)?; Ok(x) } } pub struct BasicResBlock { stride: usize, conv1: Conv2d, bn1: BatchNorm, conv2: Conv2d, bn2: BatchNorm, shortcut: Option, } impl BasicResBlock { pub fn new( vb: VarBuilder, in_planes: usize, planes: usize, stride: usize, expansion: usize, ) -> Result { let conv1 = get_conv2d(vb.pp("conv1"), in_planes, planes, 3, 1, 1, 1, 1, false)?; let bn1 = get_batch_norm(vb.pp("bn1"), 1e-5, planes, true)?; let conv2 = get_conv2d(vb.pp("conv2"), planes, planes, 3, 1, 1, 1, 1, false)?; let bn2 = get_batch_norm(vb.pp("bn2"), 1e-5, planes, true)?; let shortcut = if stride != 1 || in_planes != expansion * planes { Some(Shortcut::new( vb.pp("shortcut"), in_planes, expansion * planes, 1, 0, stride, false, )?) } else { None }; Ok(Self { stride, conv1, bn1, conv2, bn2, shortcut, }) } pub fn forward(&self, xs: &Tensor) -> Result { let residual = xs.clone(); let mut xs = self.conv1.forward(xs)?; // candle stride only surpport one size, h_stride = w_stride // infact model stride is (stride, 1) // but now setting stride all equal to 1 // so h direction use indices select if self.stride != 1 { let h_dim = xs.dim(2)?; let half_h = h_dim / 2; let indices = Tensor::arange(0u32, half_h as u32, xs.device())?.affine(2.0, 0.0)?; xs = xs.index_select(&indices, 2)?; } let xs = self.bn1.forward_t(&xs, false)?.relu()?; let xs = self.conv2.forward(&xs)?; let mut xs = self.bn2.forward_t(&xs, false)?; if let Some(cut) = &self.shortcut { let shortcut = cut.forward(&residual)?; xs = xs.add(&shortcut)?; } else { xs = xs.add(&residual)?; } xs = xs.relu()?; Ok(xs) } } pub struct FCM { conv1: Conv2d, bn1: BatchNorm, layer1: Vec, layer2: Vec, conv2: Conv2d, bn2: BatchNorm, pub out_channels: usize, } impl FCM { pub fn new( vb: VarBuilder, num_blocks: &[usize], m_channels: usize, feat_dim: usize, ) -> Result { let conv1 = get_conv2d(vb.pp("conv1"), 1, m_channels, 3, 1, 1, 1, 1, false)?; let bn1 = get_batch_norm(vb.pp("bn1"), 1e-5, m_channels, true)?; let layer1_num_blocks = num_blocks[0] - 1; let strides: Vec = [2usize] .into_iter() .chain([1usize].into_iter().cycle().take(layer1_num_blocks)) .collect(); let mut layer1 = vec![]; let vb_layer1 = vb.pp("layer1"); for (i, stride) in strides.iter().enumerate() { let layer = BasicResBlock::new(vb_layer1.pp(i), m_channels, m_channels, *stride, 1)?; layer1.push(layer); } let layer2_num_blocks = num_blocks[1] - 1; let strides: Vec = [2usize] .into_iter() .chain([1usize].into_iter().cycle().take(layer2_num_blocks)) .collect(); let mut layer2 = vec![]; let vb_layer2 = vb.pp("layer2"); for (i, stride) in strides.iter().enumerate() { let layer = BasicResBlock::new(vb_layer2.pp(i), m_channels, m_channels, *stride, 1)?; layer2.push(layer); } let conv2 = get_conv2d(vb.pp("conv2"), m_channels, m_channels, 3, 1, 1, 1, 1, false)?; let bn2 = get_batch_norm(vb.pp("bn2"), 1e-5, m_channels, true)?; let out_channels = m_channels * (feat_dim / 8); Ok(Self { conv1, bn1, layer1, layer2, conv2, bn2, out_channels, }) } pub fn forward(&self, xs: &Tensor) -> Result { let xs = xs.unsqueeze(1)?; let xs = self.conv1.forward(&xs)?; let mut xs = self.bn1.forward_t(&xs, false)?.relu()?; for layer in &self.layer1 { xs = layer.forward(&xs)?; } for layer in &self.layer2 { xs = layer.forward(&xs)?; } xs = self.conv2.forward(&xs)?; let h_dim = xs.dim(2)?; let half_h = h_dim / 2; let indices = Tensor::arange(0u32, half_h as u32, xs.device())?.affine(2.0, 0.0)?; xs = xs.index_select(&indices, 2)?; xs = self.bn2.forward_t(&xs, false)?.relu()?; let (bs, c, h, dim) = xs.dims4()?; xs = xs.reshape((bs, c * h, dim))?; Ok(xs) } } pub struct TDNNLayer { linear: Conv1d, nonlinear: BatchNorm, } impl TDNNLayer { pub fn new( vb: VarBuilder, in_c: usize, out_c: usize, ks: usize, stride: usize, dilation: usize, bias: bool, ) -> Result { let padding = (ks - 1) / 2 * dilation; let linear = get_conv1d( vb.pp("linear"), in_c, out_c, ks, padding, stride, dilation, 1, bias, )?; let nonlinear = get_batch_norm(vb.pp("nonlinear.batchnorm"), 1e-5, out_c, true)?; Ok(Self { linear, nonlinear }) } pub fn forward(&self, xs: &Tensor) -> Result { let xs = self.linear.forward(xs)?; let xs = self.nonlinear.forward_t(&xs, false)?.relu()?; Ok(xs) } } pub struct CAMLayer { linear_local: Conv1d, linear1: Conv1d, linear2: Conv1d, } impl CAMLayer { pub fn new( vb: VarBuilder, bn_c: usize, out_c: usize, ks: usize, stride: usize, padding: usize, dilation: usize, bias: bool, reduction: usize, ) -> Result { let linear_local = get_conv1d( vb.pp("linear_local"), bn_c, out_c, ks, padding, stride, dilation, 1, bias, )?; let linear1 = get_conv1d( vb.pp("linear1"), bn_c, bn_c / reduction, 1, 0, 1, 1, 1, true, )?; let linear2 = get_conv1d( vb.pp("linear2"), bn_c / reduction, out_c, 1, 0, 1, 1, 1, true, )?; Ok(Self { linear_local, linear1, linear2, }) } pub fn seg_pooling(&self, xs: &Tensor, seg_len: usize, stype: &str) -> Result { let x_dim = xs.dim(2)?; let seg = pool1d(xs, seg_len, true, stype)?; let (bs, c, dim) = seg.dims3()?; let seg = seg .unsqueeze(D::Minus1)? .expand((bs, c, dim, seg_len))? .reshape((bs, c, ()))?; let seg = seg.narrow(D::Minus1, 0, x_dim)?; Ok(seg) } pub fn forward(&self, xs: &Tensor) -> Result { let y = self.linear_local.forward(xs)?; let x_pool = self.seg_pooling(xs, 100, "avg")?; let context = xs.mean_keepdim(D::Minus1)?.broadcast_add(&x_pool)?; let context = self.linear1.forward(&context)?.relu()?; let m = sigmoid(&self.linear2.forward(&context)?)?; let res = y.mul(&m)?; Ok(res) } } pub struct CAMDenseTDNNLayer { nonlinear1: BatchNorm, linear1: Conv1d, nonlinear2: BatchNorm, cam_layer: CAMLayer, } impl CAMDenseTDNNLayer { pub fn new( vb: VarBuilder, in_c: usize, out_c: usize, bn_c: usize, ks: usize, stride: usize, dilation: usize, bias: bool, ) -> Result { let padding = (ks - 1) / 2 * dilation; let nonlinear1 = get_batch_norm(vb.pp("nonlinear1.batchnorm"), 1e-5, in_c, true)?; let linear1 = get_conv1d(vb.pp("linear1"), in_c, bn_c, 1, 0, 1, 1, 1, false)?; let nonlinear2 = get_batch_norm(vb.pp("nonlinear2.batchnorm"), 1e-5, bn_c, true)?; let cam_layer = CAMLayer::new( vb.pp("cam_layer"), bn_c, out_c, ks, stride, padding, dilation, bias, 2, )?; Ok(Self { nonlinear1, linear1, nonlinear2, cam_layer, }) } pub fn forward(&self, xs: &Tensor) -> Result { let xs = self.nonlinear1.forward_t(xs, false)?.relu()?; let xs = self.linear1.forward(&xs)?; let xs = self.nonlinear2.forward_t(&xs, false)?.relu()?; let xs = self.cam_layer.forward(&xs)?; Ok(xs) } } pub struct CAMDenseTDNNBlock { tdnns: Vec, } impl CAMDenseTDNNBlock { pub fn new( vb: VarBuilder, num_layers: usize, in_c: usize, out_c: usize, bn_c: usize, ks: usize, stride: usize, dilation: usize, bias: bool, ) -> Result { let mut tdnns = vec![]; for i in 0..num_layers { let layer = CAMDenseTDNNLayer::new( vb.pp(format!("tdnnd{}", i + 1)), in_c + i * out_c, out_c, bn_c, ks, stride, dilation, bias, )?; tdnns.push(layer); } Ok(Self { tdnns }) } pub fn forward(&self, xs: &Tensor) -> Result { let mut xs = xs.clone(); for layer in &self.tdnns { let layer_out = layer.forward(&xs)?; xs = Tensor::cat(&[&xs, &layer_out], 1)?; } Ok(xs) } } pub struct TransitLayer { nonlinear: BatchNorm, linear: Conv1d, } impl TransitLayer { pub fn new(vb: VarBuilder, in_c: usize, out_c: usize, bias: bool) -> Result { let nonlinear = get_batch_norm(vb.pp("nonlinear.batchnorm"), 1e-5, in_c, true)?; let linear = get_conv1d(vb.pp("linear"), in_c, out_c, 1, 0, 1, 1, 1, bias)?; Ok(Self { nonlinear, linear }) } pub fn forward(&self, xs: &Tensor) -> Result { let xs = self.nonlinear.forward_t(xs, false)?.relu()?; let xs = self.linear.forward(&xs)?; Ok(xs) } } pub struct DenseLayer { linear: Conv1d, nonlinear: BatchNorm, // only batch norm, no relu } impl DenseLayer { pub fn new(vb: VarBuilder, in_c: usize, out_c: usize, bias: bool) -> Result { let linear = get_conv1d(vb.pp("linear"), in_c, out_c, 1, 0, 1, 1, 1, bias)?; let nonlinear = get_batch_norm(vb.pp("nonlinear.batchnorm"), 1e-5, out_c, false)?; Ok(Self { linear, nonlinear }) } pub fn forward(&self, xs: &Tensor) -> Result { let xs = if xs.rank() == 2 { self.linear .forward(&xs.unsqueeze(D::Minus1)?)? .squeeze(D::Minus1)? } else { self.linear.forward(&xs)? }; let xs = self.nonlinear.forward_t(&xs, false)?; Ok(xs) } } pub struct XVector { tdnn: TDNNLayer, blocks: Vec, transits: Vec, out_nonlinear: BatchNorm, dense: DenseLayer, } impl XVector { pub fn new( vb: VarBuilder, channels: usize, init_channels: usize, growth_rate: usize, bn_size: usize, embedding_size: usize, ) -> Result { let tdnn = TDNNLayer::new(vb.pp("tdnn"), channels, init_channels, 5, 2, 1, false)?; let mut channels = init_channels; let mut blocks = vec![]; let mut transits = vec![]; let params = vec![(12, 3, 1), (24, 3, 2), (16, 3, 2)]; for (i, (num_layers, ks, dilation)) in params.iter().enumerate() { let block = CAMDenseTDNNBlock::new( vb.pp(format!("block{}", i + 1)), *num_layers, channels, growth_rate, bn_size * growth_rate, *ks, 1, *dilation, false, )?; blocks.push(block); channels = channels + num_layers * growth_rate; let transit = TransitLayer::new( vb.pp(format!("transit{}", i + 1)), channels, channels / 2, false, )?; transits.push(transit); channels /= 2; } let out_nonlinear = get_batch_norm(vb.pp("out_nonlinear.batchnorm"), 1e-5, channels, true)?; let dense = DenseLayer::new(vb.pp("dense"), channels * 2, embedding_size, false)?; Ok(Self { tdnn, blocks, transits, out_nonlinear, dense, }) } pub fn forward(&self, xs: &Tensor) -> Result { let mut xs = self.tdnn.forward(xs)?; for i in 0..3 { let block = &self.blocks[i]; xs = block.forward(&xs)?; let transit = &self.transits[i]; xs = transit.forward(&xs)?; } xs = self.out_nonlinear.forward_t(&xs, false)?.relu()?; xs = statistics_pooling(&xs, D::Minus1, false)?; xs = self.dense.forward(&xs)?; Ok(xs) } } pub struct CAMPPlus { head: FCM, xvector: XVector, } impl CAMPPlus { pub fn new( vb: VarBuilder, feat_dim: usize, embedding_size: usize, growth_rate: usize, bn_size: usize, init_channels: usize, ) -> Result { let head = FCM::new(vb.pp("head"), &[2, 2], 32, feat_dim)?; let channels = head.out_channels; let xvector = XVector::new( vb.pp("xvector"), channels, init_channels, growth_rate, bn_size, embedding_size, )?; Ok(Self { head, xvector }) } pub fn forward(&self, xs: &Tensor) -> Result { let xs = xs.permute((0, 2, 1))?; let xs = self.head.forward(&xs)?; let xs = self.xvector.forward(&xs)?; Ok(xs) } }