From 6890f8603053f5d8eab9483ff0ce1b85e9c0b8c1 Mon Sep 17 00:00:00 2001 From: jhqxxx <18280426169@163.com> Date: Wed, 18 Mar 2026 19:15:21 +0800 Subject: [PATCH] add deepseek-ocr2 support --- Cargo.lock | 2 +- Cargo.toml | 4 +- README.md | 3 + README.zh-CN.md | 3 + assets/img/ocr_test3.png | Bin 0 -> 48301 bytes docs/changelog.md | 3 + docs/changelog.zh-CN.md | 3 + docs/model-card.md | 13 ++ docs/model-card.zh-CN.md | 12 ++ src/api/mod.rs | 3 +- src/main.rs | 5 + src/models/deepseek_ocr/config.rs | 9 +- src/models/deepseek_ocr/generate.rs | 22 +- src/models/deepseek_ocr/model.rs | 296 +++++++++++++++++++++------ src/models/deepseek_ocr/processor.rs | 33 ++- src/models/mod.rs | 9 + src/models/qwen2/mod.rs | 275 +++++++++++++++++++++++++ src/position_embed/rope.rs | 6 +- src/utils/img_utils.rs | 4 +- tests/messy_test.rs | 21 +- tests/test_deepseek_ocr.rs | 49 +++++ tests/weight_test.rs | 22 ++ 22 files changed, 708 insertions(+), 89 deletions(-) create mode 100644 assets/img/ocr_test3.png create mode 100644 src/models/qwen2/mod.rs diff --git a/Cargo.lock b/Cargo.lock index aa32683..3c56c66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,7 +21,7 @@ dependencies = [ [[package]] name = "aha" -version = "0.2.2" +version = "0.2.3" dependencies = [ "aha_openai_dive", "ahash", diff --git a/Cargo.toml b/Cargo.toml index aff9258..739d200 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "aha" -version = "0.2.2" +version = "0.2.3" edition = "2024" repository = "https://github.com/jhqxxx/aha" license = "Apache-2.0" -description = "aha model inference library, now supports Qwen2.5VL, MiniCPM4, VoxCPM, Qwen3VL, DeepSeek-OCR, Hunyuan-OCR, PaddleOCR-VL, VoxCPM1.5, RMBG2.0, GLM-ASR-Nano-2512, Fun-ASR-Nano-2512, Qwen3, Qwen3-ASR, Qwen3.5, GLM-OCR" +description = "aha model inference library, now supports Qwen(2.5VL/3/3VL/3.5/ASR), MiniCPM4, VoxCPM, DeepSeek-OCR/2, Hunyuan-OCR, PaddleOCR-VL/1.5, VoxCPM1.5, RMBG2.0, GLM-ASR-Nano-2512, Fun-ASR-Nano-2512, GLM-OCR" [dependencies] candle-core = { version = "0.9.2" } diff --git a/README.md b/README.md index 9d70a85..23fe019 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ aha is a high-performance, cross-platform AI inference engine built with Rust and the Candle framework. It brings state-of-the-art AI models to your local machine—no API keys, no cloud dependencies, just pure, fast AI running directly on your hardware. ## Changelog +### 2026-03-18 +- add DeepSeek-OCR-2 + ### 2026-03-17 - add PaddleOCR-VL1.5 model - fix qwen3.5 position_ids create bug diff --git a/README.zh-CN.md b/README.zh-CN.md index c35afd3..c5c9e54 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -25,6 +25,9 @@ aha 是一款基于 Rust 和 Candle 框架构建的高性能跨平台 AI 推理引擎。将最先进的 AI 模型带到您的本地机器——无需 API 密钥，无需云依赖，纯粹、快速的 AI，直接在您的硬件上运行。 ## 更新日志 +### 2026-03-18 +- 新增 DeepSeek-OCR-2 + ### 2026-03-17 - 新增 PaddleOCR-VL1.5 模型 - 修复 qwen3.5 position_ids 创建错误 diff --git a/assets/img/ocr_test3.png b/assets/img/ocr_test3.png new file mode 100644 index 0000000000000000000000000000000000000000..47d169a74026b12ae81c7b1dbeaafe88a0f0c94e GIT binary patch literal 48301 zcmd3OcQ}`Q|M%Aj4TVHVR%Ip$6|z?-3fW|3Mpjf-*_EV7%9b)SqL9o`C_5!ZBrD3^ zJg@V*?%(+TIiBNq?&J5n@9QeQ-}C&OpU?aKTJJMl`?M|won0>49CWvGv$1h@ zw{!9MLs2P%A38|Gkz^N>^i81rF)0OYC$F7+(U5+)a-aY-e<@nxG z_u-E^PbYihq@=mIMRsqa3amaa;=ssiZbbIaKknWZvYao!hxpUo+f-ec z8i+rn3Oqw$5qRw1pT5mX!}j*@zdzC`_nGs~KfgznMpZ-Q`tN7&7N?U4v-J@*H?#@7yd+ke{(I(ywb0KW=T(d!9D8Hqnpm(%Bnj%8Tanp>%BZXDtln1r`R#W zyi&%*#AN+&+`m`9o{%9Pc3*8@iSLTUua=}8j*gDo_8g;YYirXxn~bGgoN1Ar>aV7t zq^4FhH|JnJWEPZ|nAl?<92{)1`1c2X?Q)FA*^u1agEciZ=Pz8C|62|31k<*$|^6t}6?F8t(IIiFl6bj*N_uNV9Wu*~;{L zBY9{1uiU(OlSJCTf4}#}x^M7GPft&4E1!5nO^w3jj1ncc!nt$jf>&I=mtU9T%+PI# zkKdDim0ZrkB1PTeY++&H-PqU$Qw>|Lp)Wsv{J4(Q71YcYm3rbwsct*p_;L2%ZIf{v zEA03w$1u3MX*ix<(q~b`Z+VpW*s){h*OuoDDtwOPrq4S&Z};-@imNuTwB%k`h~iPr zx2UCh^6Z(_+*sG{W0yFEgoOUij_PpNLrPa{T@R;mn ziQIeQURz*TzGJsAKE0~FodK8cJvSDk89&~gXH|V;)1}LoRVw|~NTf4o&KMM0Z|?2w zZ7uU0-!eb(_0YVNR0kw{BRO9n44(C%Z{OyZQ3 zWW_g#NJxYP22$L(aYNh0R@f?4iOO%61_!hvivdcznrHNB!k1kXiQ2jk$Fw_ z%IBHADe10L3Yr>huY&A7dN9;&RugTlkK+OqK%vil7E^3y4j+&3e$j-Ko4jz&R z7E$Yz0@bx^*U05wT=;NrbCUEGQL^MOFKpc{aJVP~>Tk2u28Szglgp7tK0H1uYOhvH zK~5eP#wQ{1K-j!Olayih=BlVGo4A7#$*wkpR!OPo+JZB=+%2Yq_rkkVmA8`1RaaM= zd=B5qOG7Gs{o36EE5k+V%s9g%?DVD3hBDytWbgZ6c1e#gZfC@Zk>4%1g?>~6JM3y|s7^dKxEViU>0$f)scR7IB0{BIW0r6lcwf`X{*?q@-psHrLV z65iYwbsbRh_V(uMGB-2I>NzuSIQr2@QBiTy#nbowfA-FF9>?4FX_96?)XhU5Z0t=) zN#P>N>FYC-?#9J!CiRZ%>18;*2=I{Gf9kbBhC?W~%<%Z7e=BBEq{E+TJT2G7zZS)| zm&4S4ZDb`k-qE0@rY6a$s89tda4{!7e!P`r_r3f$DcN_BWW4#KLycQ1&)lX!4h|dK*ShJbR`?W;y2` zwZ}b~PHMQ@7`5Q~wb0_xqemHf*-QcLrnyI5z6TiyJz$}UiP-pXw6ChMaSOIciN`2I zK>N33l(3+fm|X`>$J3B@A9dbC;yiGGN-=bM-Nyu}!*;El&UcL@PK^pSzkREeV_vzp za${ZU)nJiRd8W=Ya*~3ALcZIO`p=;uE?L>6uTugE${OnGHRk?n#EA>8(C)fZ*z%5T zPhlkWk$-wzt+p+hsk-`>Q;)>L;}a4}m8VfH#GHGNm0ep9rl*f!J9>xIiXyA0K7;bx zS?X>pd;w2=HGkLH8GWDUR7XooBN$}(@@Tv^-NsvbCSK+L9*5$7rO<;Ab zt(~2k|N0UyJ^e0b=KH7_u^%VDy&gqX{y8?*7);9|8uYd5OdLB_`N?zB*KRHIdTbY*RlN!oGh)G0fAd)F5ZWo2c(IZ^m) z{N1a`TVZr8ESvH7M~@!;>n$A}9Zl3C>AgD?Qs|`jMy_ZV==v!rkc*0np0~B7W@BS3 zWurl_q2u6)l(^Va_4)H|^6MltDu9FjDR0qdn$lOMDbc~~x}UQ@dxb-a%ROpL6YYE; z?AEP7Z9b*(Pq`MBmNnl51q2kVt@oi}wmm(?7N*1+o5pi$IO})S_Sto2&_9y6xSZZa z{`b$PiKKmed^C7sXj^i_9}{-fZ{4wjnqFQjU+S||D*lfCZ@uIL0 zKmTSl3k9_8eD5jCeZ0JBD4tyh?w>iH#FIBPmFc7U5D&wLE_>-RSJ9eMAxSn$fkPi-V2r3hgpy1XA4<2mfU%0t+bbMZ`Jz1VY$RM9| z;_7VG%;HdQ2{E*2OQRN7@3=!vv730)$vSAk&jUzB0DWD5JhNUq%93yP}lfV zV~n5$x;!dfXRa~xPTtdR^D5VGszk8rg*gJG$s`d#dk`1wcJW0)K{b%W;Yg;)t#M%r)OqL z$6KPhrx!cRCVu`5s$5^*nrB}5oGA{EQ&~-o2GyhU^q;=lckXNl+VB7KClUziQ0_;6 z>a&5-(Tr&D$!V&Qjc+O)x&%D$t*8<$$k4!mubhLCQHjJSC^*j1sV=_$p=|B`?LEiu zjdf-!0-3n|(RwIyo%oln%)+bIEbrgHrwk~3@j@^{kRhCgq^zo%y`M)?^3m1V5w4xT z#>Ui-*nZSq)WjwvOU8l=-hP4G%+k->>-qchnaMTm{`+{2$EBpQpV(qE@f|pzGWn*w zTsq72^-J|ppVMyc?xQ$yU5|bxNO_H{tt|A;-P|SkuAw3H^{<&F5F4{x(V;VjkM%k( zd3vVf)-xFw@S)arc6>VPGOA{R13KfE@xgX8zdm#w==k<60Vhb7|Bb&CX#@|>+$dsW zo5P27)^W6zHK_2NMmY z=*5Sk$89{9q9wA0ZZ=-9acgQhwlk{NaHLR1r!H^0X3Lf>z`EzS?kI6b3BK_0vAJRz zmta|7V8NEHRw00PQhJ#p=2dZ$z)o$G&?hgV8^_L`JQ?1;u5!jufS>=2Ba?2-a*ld$ z)@K3bN2*z~u0u1Y_A&6!#d5Rnnj63Ak>M1t&Eh~iKu$r?_QF;xc!5pKjzoHR)MYQL zxI>Wa#_CQvd#htWL1OmpO22wOePZg)GgnQJ^vrkqBAR1RK)!wZ_RlXbvf-p|^7Zv) z+_fvEq+~ZX?dK9#9>2vO)XfhMQ!s>6qRNJ(rSV8OeGVAyc$TJ@eNN=aky?}|QuOJj z(BNRMqeq#cMAQLnbY9kWbv+#GH&`FhR_v&ck0c|3ZPeukE>67aJAeLsz}>ra_*09~ z_Vm9~Q`JZNd;-ZRsLoIH6dP3forrxZeJxApRih7uTxKGWpHh;HZ~xDqZ|h=zwE&q2 z#_S+*ad8>spC|kD+;ppyl+@~U)1klf^Yz%mr7OVlH7=EFA*gZ#s5o0LCLFe+0-s)8 zUA3;k4ITy<-7+;ZGgE*P3{bzR>A}Ik$B!RtF8j!EKpJTH{pCelp1JHc4w-TV)~qqF zi5_vyBpG(&SI%|a=Kc{UuFdO)%}fodQf}H*U00`^5}{Z9?HjwZv$F}yeYeTpaQXbGAyJdN>Q3h1RXPT!wKSxa?IYa@~GUF!^XDxw79suYzclwFi5BuhZe-srB5`y z1}(ZcLN=kT%8l>{q_w-`nrto@+FF$_FvZ5 z7W&9YpuY+xCVM_6$wuPI^6cA33GnN=DJ~@?<)Ngf03;9sv7vdsP@g*+BcTzc6%(_B zqLOC%`tm~=UrERxk>_%as_W~i7AC*d0%^8Y_+IOLZu(eIwk7dIZEK3+8E6yI*XFlj zDNx~$8x`4AuW4(iN0eNiURB$fShska>lYgM+|>L*{qT{iF}=i`oWM~&Z9vu z{(9`9oo99e>*ltAVp8++rGO~e(#5vW9ULN8rmX=)e$)hSjZbnj&?=gb?aQz}q3h|fU%IY00=e7A(t!)7;<@|@P0N@0fZ{PWf1-q)_x zc698VoSf{9BwhL8koK{Nozw%|l0F4c{pIuLcjcF{WpkPZZ@jQ=x%d3J@IlkEXnd;@ z)E(Y%C54MUMO?PFwp+Gsdt&(FLX?5fQ=Bm!B_&FFdellZsWiZeC?#&tY?Lhv9)_u#7e9XZP#c%)ySf+!E(e{?Z(o!e z$Vp#c66;ZCq3(|mKX%f$-Pp91`0BUAGvQ%;Qc{o6v{aRpPMbo0f1vi${>^q)D-@PlRl%D4YY+_a&f=<)k+4%&R zGVU+Iv~mnz{OmcUyS!xwX%i(S-|ZLf!>7xx%pBN6Ma4jh(?5g5B@QC^2o<`tqy%dT zjp==TeaNkyOiTx6%G5dO?uLY@Y&wob`}yaO*7EW)BL|08W@aYT3XjpV!XLxvdF&da z`IVG(O-wX>SLV~fYdX~K>@yIhX`Mq3l4#lkB(M^`2 zjQ#xaL%#dDDGzrlSbKYGYqC|t9pbqc6^XUCwWYkdw!qM~5qr_zS4~s%32{d)vKyLH zQ&amGw~Wq?c8H3$B*`AS@3p?Z&dADYlz!FF@Ns(cv&Kg0Gxol(uwNDzy}1Q)^#S1+ z>FDGb!h`PYJ)w>cZo)K3OoB!&vk0G2`VeE%@Z{G%eukdYJGBsp#B?xRmK?*vx z4mK0PN0X8`_Ve=Yq(9^2Bn0}ZuAz~3;|6(|_Y^l-a(=$Z&4`GY$K5es_%HHu^YDa+ zjdW(}PzC_g%PT$6mWi6~&fL6tb0?02fPesZYWt9KCGoXLQ-ZWE~nLI=Phg(o19k;{RlJOTV8z8$#Ay@}@D{re|1G;Vs{juf7jQpiHe9MqXoFTKGjM(jb$5t9y6M)nbhcX zu!R6*?d|snSxPB=E%^0oDbKOa@G!MW@3j2HU)MBPV)w7FuLqYhDD&W_+P)_YNceig zU^Uc2oXwj4>Od%S+snPD$O+N+DmOa5`un%jp5tzma-+>n&2dKr-(Fi=7ALqM zPPnTD$Ok2fl9I9?_W;(Z427a}1)|;ElZxzXVq#+XIPuQ&4)edi>>4QdU16eS71J)u zOwQWIDnwzlTv;h~^q zlgQDgjvh6vaC&gCNzVQXND*-=OBb(Ry}DcKvJkElLM;VUK29z!GP%0KZ%SMsX`~OL+M3-9#Cc zy6u-M(tFE1M#;edP0BnrV`<(sHPwQhCa0v3LGV;kQBhD-q`>#pLbLt5Jm(0aSboEq zii&EaJC6g0Ya8j9#|RxRE(9bI%vE6@gFJVX?qcB$Zr_+uow(eg!x9oIfL5iZ8xUj( z6&;`JsNaWeEep8e0R=FPCnUjvD6}WNo%e!~)wgi5Hy}}KD3t&rGSC>)P ztURp0|2$E5pg1^;^QRL_;lC~|dgRCy9(!+zYnnoWfH8AG`*QMWsWX>PS)ewdJX`-h zA|{5?C?7MPF3Px6R495#%lQk3A*saETi5xb`4{~u$c%*hQe5nrUM4Vr`o3G*TV#EC z{1A}AjjzGETaqlFy?iMFedySRUzU$@MJ(?NpTu)@7xa2krwh~FCMG7@N?fge_Y`l% z-CSFktiwjw0bU|Jft7{dy#_w*-M5TF!55LY!QZjHGDF8%? zPl%#%c@9X%5})yRZcZ802J%qYlPCLve_*}p0;zYD_z%tTM;i5o&&;Ggx|Zk*OOWZ1bgDk|!_?D`@F>cPM#-Hhpz!9X4s zC^vtnrxl=-x{iN3I9WQD|4g7PGmBUI@D8#sJ|;zWQ3^}2$4b2Z?0q0)cmurSJZu!g zH}L9fZr;Yk!~_CSW+ZvYtb8~4d=1RIpC7%+`XwL2J*cgzxv`a*hSIJpYnQk5-b5MS zn`r$gf#f7`2J3e1r{+s4M2(6)WKNaT(vYGUD&q28+WX3t_qvy{xIQAv%2f5?11*la=oRyCyIEM|wX}9%DSMlX8ExiLv-qa%ZEPya-hKQSqQw2ffg$mH#UXkQomy zV+oha;PvH$U_|>~>}|p7Dx_#T_kdEp1r08M&xJEWp%}95$Az~|qUWnfnvcXb8^3g_ z4j`jIql(zP^T7Fy^|ebNs6gVKg?dpkUGe~7Aeni8Bf_9J0rPOt(Ucq|baWha$Z&7L z0*5}y+N@m#PXX~N06s1ESzhF^Oh)YN1JLmwX|P`-l%Kd{&9$)b;p?Q2vff`H>M8hwgh<@lrKmFR1smElQ7 z8OE=VbqS7)k2l2_S7?IIm0w$cCG8`J2YWjHSarWv2J0OS(>hsoF z9tI1uw&w46QN9yR{9`uw-Y*>vc9 zVmjH!869gnfA#&n{g>R_Y5)d*jExD*M7%^9kv>(z##TN7#Oi?qpS%{tP3*jjOC4`Iy$Qm zh-fx%t_F<{=TW`Wk4?Nd`Yfrn&_=_g+?$?y`<|*-eSTZH6`X*e~bp?ZZ)m`J+MbeesB_K0Fd9>NI@H0tiDuFUdG3 zc6}8m&>w#c4FyBC;6fGxtxWWVC#ipMFfb^HQb}1k1)fWOetrO)cU&?J3cx8QNfYs!4s+jzK1g+NiZp#y?V4(0)C1)e%c0* z8=ajL9J~oDN)7#aQg9j!L=o=7UTJTFWkIcaRkQ6OPA*|S!7Y5AN<~f0BPz-O_k+NM z$YNa@c)R&G!X8DZ$0Q|F(e)9E;`gcv!By$k17Y6=3%~|u2dyBE zGo1X}Kx(~}{v6m_qT@Yam-!Ixw_L4%`?mjG6c2GJu+L$ya58RjU7ock^v#Nj3Mj$X z0eGQWQ>JLs`1|{xc7fzHGB(ByR0g!}|L2L=D8q0;qG+S7&L6atne&=&m?+1PelOZUyy!KMsy%1x0Yn1a=WO zz%kmZwzhUv?k#)-E0Ap)f+s~6zn**UgLoON{J`w5AmUMR6X4Xlnm>FffP@c7!^6o* z!ZxR&rPX`%40meR{v^CnbZu?TrjhXHhF&Urj(@5HpspI6Li!^d@TV3bCrC;Nn-DFv zEk%)08+@6eCp#F~p_8Yaomi92M`?#lylT zB_u5^E$0MFA{o|-(1UZxC+N4x!#zB4O=7{}I4|$Bv%Y9Aj-|^$@tM|+h6K;TX+Y(r zJ_cQbX#D_DZp-i3TnJW)8b+9UscC6Z4<3ZzkrT2vo|Ue_1_X-qd~+>z^&4nRsi~=S zXx|X3EU`r1(?7#|FBLC6nV_>lzCiDkg3S{t_Kgn+*LA>WP=v_(qXsom6Q$r8`if4{{pqduHzJ6mrj;mN4e(qdf_8RHN~%fd#-@s#6JsMw{Dfcd z?k-n`>1&CAv&JaUv=kH+&g_q(TSi02)Qn@qbi)VXgn}3HnZ$2s@wikwoj=bB(tXxK zW!C-Oy|Z7IlN*`l)%P(hE-s!29_n?hFe-MauB}ZoYAh7ceL5)1|KdQ={jTcRq8W{! zN0Tc);68#-T?v&xSls3N9wZV47F5OSKKUE{*F7xwr@h<=6vJNvp`$kq)ST7PnOdB( zD&^2vJ?rCq;#+U8z;EmC(BLIpzOU8^4#!&8-Q3l?{k1<2$bz;b*K{S}?RmfueOeAV z^><&T`2|%m#S>2)gVjEsWW4=I9$S;1g@ylx--%<#c4KF$;8ay4Ur6E@z4~@TQrc3y zs3$xkV(8N#4E?PLkwCf16``0*Uy8qav(D1#>r2`{-`^-AYgk&B1bw|L`FJ3g1~Lb* zz(_lH?tJ*@5x1-?hgJRUr6(?mqUy49`Dj|w-hWA`k#(y}GZyytOX*#fKIGuKO0ZTT z2yDY8KYaYyy-#B`&uyr27sx*~@~XRKd`fzCeAfm4R|@uLr4*y>zv(_cMfR_8y|8 z=Q@0t9(wOC8SRgY(moDS&pvS?)WrVsHovycPXc|h~-arM*e_wL;j@maitKx$P}(^kYRdf(tES0hXH`s0TW z!GvV;;lqzVf6VpfvNg@j_7WWmg?lfmow-p0>nW&ue<4Hbvuc@1J=+4E20wn-qJ~n; z>9h7huB>)sQ;I)sTiHVxp(ysEs`0wPOEbgzp)s8yqyW*t5PK4Om`hfgWpH zyy5{DvOh7#FGcppCR4e#SCT4zH?%cUUpeNG#&-oUjye@lWe0GC@ z^gsX5+$`eI$w$~mvTL&)yiGa6@Qp-J-TY^z^389IO(r!x;Npj}ADo@-nBLOpIIJt5 zD<2Pf0U*ELVf;avoBp}kn@08zXY^o|QdwCII-DHW(D_zDw(h&Wv`69-G z7UyohGB@79-Z$qm(D#|?IP$kk(?h*;8*YYK;b?Gbxq+j4#GMg-Fv?g-K5}z#|Jaa~ zC7=THURKtderwBm**+m|S+6&u^h022ePCCeKd$Zg@C~GM?2&=tB!3kUDMGTszgnU< zlz5Ia5p4y81NP{-U(9ZDawH%TdJc{qh=3AlYaCj*@B;5ocunphU^+AqB2Nli8QB|o z$VMpR$Qp;jv$25nsHUaW0I_Qd=~=fHTVPY{GCqEOd6=IEv=X-=@`y8i?%3ATl0$2M ze<%_F2#^x#qbdkm_}WOQf<(zWa-q3uupzR*vTh4nnFW%@B=u)X{(xxE9;zXE;`ksw zybCpz5?7DZ>sEAs!cRo_iHPG6aUsa|L<%>cJ#&sgs&HKaG$7dMjO*JYI!b7ZP!Ea3 zCPARFYdIOh2?-fl8L$x|lM7es2qJUA#L342ft8hE{7M*5YZI!B_tmRzOHb~&pnVE{ zc)Ysm!?IE2^yTcN*BdnN0wkRYB#fUG4ZYEa>x0}9ioH?(<_*$puB(f##6wtJoEqtP zwu5vj)ZHJ**xufrfCFg0`}H44v+atJS0)fL#Eo}2c-h&4P-O^K(p%wcyx*6Ho13t8 zK{HZuf9T{1%@*yAKGRrONb|`V88n1diYFWfm}Z3zrQU%}O&Qrx3m;`uxDq)!nQ%hk zxEPRCeM0&dsNYUKMVqnM_X{=v8GwOF^glpy7N;6Ge*FHOQ{+g|1OtUE6f z{Hrm+UQk&0S5IO~7t_V>~+RE9B=-YXZW+>%}gphDJ9NW zhNq_<1F%JpnX)AHd{UcHJR{p$e#QMC1~2RP7H%IX(ffqg0!+6E0u*ec&H_~iIfN4T z;qAVe;*f_NK#LV_{(dNdEb8zkVHWAu?FOzc_}N zJ{p~nzUmc<3wmDzKy>@3hnn(Vy$XHt;>fu?Gg`=;Q0gpE6?^$<8fW!7H+&pQW7YQU zYSc!!c=$fQ&eIH=w8ji~6@3GP-DJry;23BqJ?6&r2By5lAdV5q*1CoUQ0tv@qaC5w zE${jAgM&Qo-@l-4+v6ek&Vi7wtQZ37so`FC45Vcv84;99mJFt9!@Ib5V5DqGl9Hx&+ zH1>khD=H!J6sZUX`ZFL-$f0#WQaQ#kIy(A3o|jKKF`}_CSsJ)i6}u^3OXbd;MXhT*1xA$Y)fvtgYzG&kMC+4=58$LQPHm2qMp^IMHBapPz_*d#07tj&1h- z-McU(&_+RKOeAf_tpHFzJUko-rongUP$YJeBJNRiet39T2hwAB%P{qFlz0=dv12w2fYNsAc3_W*u zbIZUpR#J*iNH_)JGwh+OYyK(EJg?|<#+WMj=`xZfCGM>L$VjoGE-v&wJZyzR5*`)? zJf<-}KVM}1o`IX2h=m&(8+X7NJZ=hZt@h7sf_g{G`}Zku8+Ve3Ncaf2BO^V%d`L)0 z5uC)+3pkFgsB&1UGj498m_^Xf^19Nvuu0F}QDB@J&n6Ss)(Ml9$Wh>kaC2i<>li)N z1tq1`g+wCk@?_{LI$>wNg(B_`nTQNX8%j!yY-}3He59b4eY8c1{55vXP$u04QL_}> zzjk@&6bOG-*+W@bdt1Yx*Yw5_<=`1p9h zVgSvSmROdz^z`h~-r5)FVjn+lT3bU~CznG_HF;PlZ-P?7FCudH^~a{BG&qD?XleIj z)F-Y4+B9WAb`cTU0N-?5AebMibKLDG4K0U^$${pkCXJ^*@-+L9MEN)EpyGlK zE_Sg;tk`D&4-z845G&7hxnwa@?$nv#misX=F`8wBeCvzfO#`WGL-#CQ0hdexV2HCN zjwSK**H^Eg676HURVtdA;lr@zPn7E-V2C)f`{nf$?yr>DB%C=ntB(qG#fB zf#QLK#8j931N4leVfPyaq{4I_tfR=#1ZY-7vK$Es*~X%vlpgp22Jx6hm;_8b=HoB9Y8z!0l(he2j3u4^}O~vlyKZVBrI%TaBwrpw9+pD=*S*t zL`jKQrAJa;!V5kicx3&4LF$C-}{ zWfmb1ZtgL}`?_DE7NaCzO_zM2Y{;gah!fobBk<79d8pg}+@2D5%~W-LocNNAqSfkF5KiR#_5e$t3>0Rlapp1g&fggC|| zooDiFyb_ihi8{^aK6@iW51e{*Ns^ErvA?{h1~wr>f0M9;sshL`nRZ8AQq8M!X>pNo zo#xtpg~gi`bu`wSI6+XZapw70F#t~v2jz7PN|}b zz|1J%#NC*fO;CwOMn?%B424P&>abv!e2l2UgTH?-3AZ$!oeyKvYpep+f=fn*?*%P& zPvX^X;Dy{f_;YMzL=Ne5O?`TsIijw5W64nKw~z+s z6G|doLf2l!1rh_Ch<``L#y%@_ZXX|Wx>d3Ciw!Ut;EMwAw_%{}7Dgn4@mP`_yPt#C zk|E#qT$@8V!-$u0sX+M_llRi}eyD_m1_&F34UeCg1q`FX5aVt!JMG}v=$Pul?kiWW zsCUHHAN=EJe~pfwp6h-C3BrG^7 zMU9mUfHjwjYd?h>skT;V`KXxMV3mg0rF~&M{1S||-vKGbB0{4e4=^SC_nc{5xb7X`D zPX%mY0PPEXo(kYR1Zj)&ZK=w3`2XTVL5FT}%YKKVtQ3S6Ot?3@5-~Y+cJ%9on?0Pk z?(|Xp5V6Q6ofwboo=>{P92r{>JRtmv%YVMp!i9*7P$Y6ZeoI5lm{yHhk;iV_#LLSI z*^Teig8S?#C>XG)@_Mf!-GUHK_IWmKNiNQGrvz=@zN|D_BIyJ|o06NGTUcS^nX6u^ zAKcRRF>|fJT)=lz_Fp5`1N#xE06~GAT@6eu$X*q5s->-s7+HI;bnE~<)Ovj$( zt1RD+JxxT60oWtZsTyxTX-dQ8pw^)9QShzJIa*r>5d`b)eZRX?$m@MBzjEZI^S7HQ z5msnmqU>o<(PNoNpaqI>$AQmC>dg9ylAbp^jvhQni$g*jNK!wtg*oRVxgV1y1Gz%# z&46m zOHS6JJ|X28D91@cmRVf#rD;|YxrG7ukpO$)%oAyASRp;t>o-g zS$6H(zw?=>FQ!RrhWt_gp~RP7%UV`S{1cPDHTsvbnOUOdu$e>xni+OwNH}m;TpA(Ao(^UVJ1%MF^acuxHhvf6Z>ABCxRo56h# zhqPz&$*bh`YaPIhQTQ0ZFq!CE)|QqE5QGolbd7y3NYh-Mo}K*m7=Nz{;T-I_{E-RV+sOQ0kLa8CG8er1La!=|XVgfo*48>jFd-aQ5QH$VIP ziIL<~lo}nlhV=A+w`ah(?XjGUOiU`!pOPOw+|4au`V;XHW!P*cW!zC=q}HI#JCCvH z%|!@YNqZ-26Mpk%Dv-flWXeHvZy`=82jCe~Ad6{etV|}pp5S!|ilxFEYOlIZQR>M_ zAUR@M6=d(q)vN6o>M7ln7c*2@C~B}bN=*rVEEjHT?Az<32Te-uLN?PxEgl@uefkHc zC6S%R43m82+C1Bb_~ZAHVjTs;JNKTq{VHK<0Pv+^Gjpdx>-q3uKjLEqTLunq%8f+9 zA+WY-nMX>xMr=E_)CkIos0=XgwtUx|!%iVCr;Se7q)1g`YovDgnei!FIfl zYQui)QovqWKTUKf9#Rpe{{ZVZeqTIeXzb+VWG%sjm|_Igj=e%V^UCkE7n-~j$~5BD zowCBvT(R@FM~MHcAa9Wd{@sQL5}x6a8;wwISFUj^en}Ha87`C%BVs;f9? zRRw^UG>8Wbq(`#;GH+3Ln=>atKWPa^{f;Z&;3XQ@!##sOs%QSFrcRkczgQ&5^FO+mFS88_?R zjbl8EQJQ7bF4`>*CG{=lb{FpR+{D*d;`v|_Bn8GJc1x)K%R87g zyN|3l4gpdHxg+2hL`WO+WDppxC7^5j%;PClW}qrer#oLaZwRxl$x9p z0%dSLYX}n+CLS@Af}(>Vy@L>>5pfJ7yoe{}NQEF*0+k6_jZ1lVZr-{@xI%D&2OzBv zJXK1c5*HIgRIp?b;x9&uG0m%X@?og=boO_Cr>B|Zv;?wYIVsS6QE7S;_xbP z1O`7v;Zuo_BkDGk`8Wj*P>#uj^4VBDlzo^pHZpsbAbI2Ki6KU$`9OabW#>Oiab~5h ztn}G1KM>$kW~>uFb%Q;XJ2KV&r8$GKrI>?Q{pV}U5$pv=#SGHO>9#3%wM$t} zuAogy&G==)!jA86c6fcUVPIx_W&0rP-L8#hC8nF-%KYunL06$c1t1;dteSsJS?kEwr zcOKgUma^|SciOXkaonW59F?a9-3fEbNJDOqQlrGjBQ*LG%KUX?yD|DAuc}I|kaD*` z)ldqC2N#wGr&0fSrFO-gMkj3+j7Rec3#(vgH6M71Fd7j`Z1&Wf-a<(ZWGizLNK zlIt!fzm~#U2F?20*;NWHmH<&UpWr?XHDNBf2zNJCxlpF*^+0Zbq zU7RQ8y`W;n#ufe7v0t=-&9L*c=DNq~kL&qr>ZNzF0gdtM=yW)k@15wYtn6X-`6sw& zj}Pzqc+~OJ?L^r6^46_?|6G7)lf&!EDkIIIDCD4q#8T~5h6NYJjvcxUkVGNObhlb| z#u20yrX-uuZK`LV+y@*2e|P&~(tX?q{v{yqxg_NycsB(?N1>jR$S)x}q`hd9Kt=n9 zEz4qiTF8#t2D)3U_xxYeIxJrw^BI$v%52)G=zk7mq~~kn3``AHb{?&a55OH6wDt8B zm)or3TVSuw=?QHG3f?2@w+A<+3MrU~D#Pl45j&AEU)PZ=&kj3>$i={zsqO9E1H*&% zV0-1gNG>LEVvzsX<=?xB{1j%1Fac-u<2m2PxZU#g4ft3$D5$q{iipsWkT71kYG|4) zGqteLfXFzex+5^0z@NZ0=aPKRec(Is+u)~%|3t`&K_tiC(mTd09~iRC4W%(omWy$V z_==q$rI?4%G71{BZ$>yyM`oO5p1!#$EB44Qx&6SFty@#l(rTMSb0s+@bvV)J4H<&M zvT~UhTm}MZ2VmV`um}`etXy`?^K?A%PyhT;$a2beYF88;faTAxyzF!(S-e?@ zj+M2iHI+n6z#|jYg-0v}QTrJDGR$qPnay;)r5fs7;hcmALrhQ>+BAJXo(J|VXxcLB z>;C8a2@DYE8hftl^mk?J6XPN{(t*(&baW)b2o=0tsrCJtX;(0c2{Dom?xa&T;G@U* zh1f)f*=ZY(!@|Oe z-ni4u>xTi-fx62a&oZAoMUaQ81tpWt48uZ znWat>t_M8Po#TWaizLeM!hWL&V8Gh0uDQVIxr)OOEdVN5f8R>OZm~dSM1Z-Vk&xFfOhdQym8m93b+c zPc`Fi5IY+YTjDhYaeXVdx85B1SQ!&?hv!cX=zsRb#dWTMd(uP-8>;7FREn5uDOnl_ zU4psCeHiKsUSM~xzr$`kXx8g#PqFm*@8RSOTGpBhQow?|M)oG0iM*HUGZ>g+RePkh zm4YI0fk|ZSMsDuKh-^2<3~Lm;oIwPk2T<^6_k9Q*#1n&WWM?&1^% zaYKM)Q)sO`jwkGsi0c&8eE=dyM7(Q(7)EyPedlxM_vmPNUd8g3B;WZsqq#$e4mCh8L%j3`pb9a$3rJaw zbP(OHUBQsmED%428`S{Fd-v{LBKv+G!{F+FK(>T*@}PSX*=uM)P+jT)!08znZZtP* zuCA~A*)8tC$kT-NCH@TdK^4dtUIu{VwmekAzkqaj+tfCQ=%L_orxT^afsBY3Q(&A) z4#7B_^B}1D0kuCwJikKC2DG7PXO95EqsMeET;}s9hLE)+GH__vhmmn0EN%dWYD5kH zez8r$V4ol1M7tuAj(wn#`3)k2VmdpC_;lj?pjQy$51{H9$GAK2X9n=P1!PN(5w7{3 zJ==g^oj&K2L680X!gj}hVv^XlA`2iGTzI(XOL$A!UO1uk%RXz^>3?Tt6irOx6;>+n zo&!LZKnTQEP!%B!#MGEyC@h=ymd-JLwOww=2W0UEfG{};kBQLvpxwdFqBap`M*c(* zDCFtsge|y&bBW;ogiA>54#W%*h)Z0G@g4ri4s}Iz>vvfGbU^BQ8tXT2ZYf?pnTS>7 z#=0UZE+Q^o2fzx2gBYq6c`Q3VK0YYBv9yDTQG&D}>QsaGUf}+Sx3a;)`0z6}CMFn9 zjClVD@!L>6=%E7~6y}>2hch4JiDSTckdh7gF+?S}kcaK_TR8|dOgn9YP!9b3{18s6 z0bc_jp^AR+z>>%rU%FHUg`3z<#O^^-BWDpYC+VQj&_PUe5aS9&2oj}=bN~L}sTWYf zi9rcSgmgrmz|QFhkwNz$W|{Coc!$|@F+V}cKGb})zxZ$-jQd(U%_10gS3O3cTA|7i#yb>kM&h*v zu$PInFdX8a!^2h}YeWhH`{MBVssM+}3QEWrD`Eza=){opx1%@fWYrcHimD8Q9}xLE zcnN=z&M2Jz@b2AtA7FM?a2|@#+G=9r>i3r;7-*})06gG3k%I)w3;>r+^WRt}1wirz zEbT|;-llF3WE{NK=dh<4aG3_MghUD!2CAR<{HeyfE)gemg?Ixf zq%i8O7-tU6ud7TpBQtUOVk_P#fCpW;{N|Bm4ZZ*aEF^AXaHJn)04>E54iqsbt!~=L z_NZqTBEHm-dAQSMU?xKG1 z19r46_^;1!05U-Kc#jF{J?!lEph3||Nm@7~lmXB$m5?|t@>{#w{^`?GbodSwjS(DM z%7Egbo4B3ekPrs?!xKl~v;8vB^IKoCWrf@|4&ii7jXO#k1%D0c3(aeSFW5YP2ZRyJ zg|+27aUucbmB{m0KG!Q2yh51jPoF*&J6}aw##SQCPC|kqCZQ+3ya-K%3xnz z1*Cx=!JVky(*lPVc^Tjak?w_wuMnJUY;44AKkgxy!$sqf72gjHr9PQ{_9~=$afi+@ zOn*OI&FZQ4MfMqog_XJuLg3v zeEC3XW5k4%$4J%UFs_ey|5ewd{((x#^?h29=wFN;KQpBAw=m+zFdR-FnIuGI2#XJ| zhj~|DA5yWger7A_47&u%d`tLTypR{uJ|F_8?w8gKgXafprw^oH78u z#Yv1%<4s2pgKrUG24qGV+1XEFi|of?gZg2++FVr?@MWJb%#&FhRW~oMXTZ1I+(sA_ zdLjS1a4~np=gO7if_LcXZES7tLn*@l9I<~4VFM2YES z5F5B4aVM`4U6c4QsOoC4A3;3z(za)h`Cz?@Kr_z?MHA{gxg6n45c?x64E0nKKZ%id zU0o&YBq9xheO-jXNztQ6pY`^d+`Mz=DGu(=sDXz(*)va0-VX~i5Us1PXQ1!;e3V|# zdFsGSL)N=j*Ci!*IM^^w;IPMqZ{}9nEOq0UAu`vww2X7t#Os29pxQCTr3!WMDb6e5 zy&p6ye~3vN8@$7%!}L_A1D!2oD&W>M6dTb9IVD2pSza$;Xb25+ zaT9`M(|KM*a8>(NWClEbmX>njbDYhN0yAQJa_K7I*F6p2(L6D5-rCcsHcm6 zdgbu!8iH7a)HYs*7h5Epxb`3{4B6sLw5sM}BDDeqwX^IMI>2o!JzZTo`ooe3xEq_A zcoY>C|L0|NT~{!wmQ=AeZ~V((A49mwAMDeQ*4&!Y+098xPYhdn9saVXYnMc3_1Y zj+af`jf~vkE{$N~(-Z%vsq=vAd4K!=*EmLIcDC%YLPkh5tfKXmG7>UMcAQ9ADKbh@ zp+lvuB0^S38ahTQ%9dGX38nFWUe0|#{`c>39{1xMclwUcdtBFRUAw$QL+|k9f)>GY zZQA%CELb=uaV?yO(D(AV(5)nvx}%eGY-wd?cD?DZtgNgIvW7TQZ=OhSn2~sK<~Rh{ zqY4TO6SJiFLg+Xn``h4@s)<^8?18Y2)wbfRizKZ&Bigff)VC+hO$qfkyUPZOrP zDQhhK1ziPE3@qXs^8rm(RW?5f)MNdjVbczJ&}-SJFE=BI!IZCM?NJU1exbq5eNAFq zO_W{Qq;*l_w`L!oa0_LlWnptw--Lw00!Jp-K=>cp-%6GJw=XFr61vP$(aV2oCT?&f zBpL6$QeT>4b5K>Dd(LVU{ZRHeZj9~t8t75(=50~CZK6B(8sv##HR z3T|Yt_bJsAl!&PtD;^X?MntsoOtnk%KrM8ST#CT4mpGQ$eG!PPPL*1=SxZ53qv@-d ztX`dN=H-r5?ceRDs#5M^Bi#{E9ar`%Z4z2o-Y}-P|MLc=&K7<9&<&nyuoti03BGPw zxKU&5bBpmGn;aR!9f29Rl~mg>*WZX~d_n!Zlm(#agv#{~!&Z?o-o|Que8h0V1TAOv zQ9xA;oqgN#sd+e?4d+XOGH57Q>lOe)Ny34HfKV3$!I`J&Jojyg-qaDPaariw+4TTb z2+zOLxp_mhHi~N})!9co(S7OR(BYJnDx4a-&K4JKo{Lbs9BERBO)W`#x$1aaTr)7X z91r*+g&=7dTW>%xXt8+3wZX?QyBIm2H~y#KK>E(jiNhl8K<{Y>4<%%*K!Ff?s0%v5 zmuO2)B_@iyDy*LKi&dje{#aR%*qcXSolY~aXxXV#lMvI4#Qt0f1Vwb^JeOt(Ol?+$ z-!|t&i3!3d46HIy!#}@Yg3gbT=*c9D0G^H`y-wjdr`UCOW^xoY?>g3tulfdzDVGWr zq|k@_!A_p9gdrmSdj#RYt?4k(-K3^h7%EHAnc{$tw@qn|BWu;D&XSg-QKA_Zvx5C5 zPlZbH;Alxd^;o8?IDFBWG!=vT)9|;n=ZbMW+a+jE>A8M`rztlbd}>jy)TaA`O;Nk# z_vvc)2Oex|9+(0HIL`adT(IvgUkKpQN#npiQZ`12;s^$h?wzaz4lYy zk(c0~KUXcO2DRUiBWta$YyfB@ zAMd7E|G*asIr0fQGOIlWn%n}}M@F`QEg+6kl%=;%zYYRRkf;edz8kr@XK!V`;I!R2 zbW9@^l^eK5eSLfs^f@9FN2C+IcAVqH5f!6oGRh54WbaF!GYE@r(AHrJ8kNLS2>_Xw z;e3>;o&f+ z=0RUBeb-Mi^lxsTnESR)J-%=Q_3PYH{wt!}b?&@JEX36KtJbcy7@csTPjGJT$h?tr z23f^@GjjSKxo1Q4JI_6hlPcRmZzV~*C_XhVP40(gtQroHR(Q=$o~4Az9=x)$_Vdl#6fN_bSIdBWVlM&p~cz z7hyudF-1u~r@eRDN#EZhCZPR7Kb_boIQF8|yzz-1i|l8(K`Q$q6g#1+;^DPt&kn@E zpck{86n&>zl_!Ko8Y%CN!+}L>l`51k2i?ry;XUt0nT44G)^m^+p(3)38L5~2G%9gB zN+3)E`iN-Swr%^1l6qE!Mr=T0$?lxn`?_@QEdEUJxC6;E$$q9ds(M~q-3^=x@rx(# zco^mM`ThQr(7iucOi1^2^r{4B)Xehzx1RruBJSm#XvlzV`kgER&M=!d@6McnW;eYW zvFJpug45tD?9&3OQDs--%yc9;}As;B$a5jxS)D*%rk5w?A%-D6~OtEtU8R+|L7 zO%Cp0enclGEcI|{y=ztT550N$>Q!A1DvWIY{?j9lpj@h}5b*^_fbr2$zWxE1uZ(oa zZ3IT=F>62r1!jvt-zm!HC!+3Xf6(u;IV&>y)8Anu>~HT==2={A_PQ)~nz2s+))=$6 zze1DgmTkg^=GDF<#ga8^h&Z=QdkS9%RoY+?&QLDy)R4SdUE$ zRv?!w_Z@i1FLDuDcIncL%S?=&;1QCe2W0KZx(%g_Adq4kWhfAWz!qZc&dAK1K-tu` zu{!jBw;;IMi<*9dhQHXx`E8QpzmMBL zw{+I8S+gxDNR2S^UHrZ4WIZ7DuB&+T=+S&ia*^VQzxv!H_MIx`8+LooL4%s}HE*MO zrS|15>;@R0$jM8AGm*8%1Dxa3Y=rKy1k)O1E&@<#S6YCwTf5L~Uk7Ib;+)7bV1uSN zJwZ<`y0eMY9I!B>ke~{F1RsA5UIspK*=&$Kc|7dQJ#i^eURX=Tlf!z|9F);uROwX^ zxsmKdrG1yzMf*P9o}>`m485il1%e3!M56;rljM6=rAAUP3-05yE1i(P$UP0H_!7W} zwshivcBwojjIq~QQ*uQj2*EYm4k^Pms&k3Y6)U+xeV}0wsMA%(1V!LZ)Z_Yo@#@t! zZa*pR1Zd;fWo=1{lpQ9KysCSJg*Go;;1p+eK7Hnl=-wnwn&UE+1ahDk<{Y#eJ9ex9 z$zW|!dxi}y4)yEi_G^M2{>3K}<0n=h*|e0FJ^ZIiPK-EW5E%?OVt&5cRAb|toCHCJ z`)_AuokI6B)~zbA?OL%(Km(Iul7FBzkitzaBJ6tgsUYP`WpycIQfb$W-m!c5;C2zO zEz>9p#bt!Z6?nsOV4TBx*prmf#wkP9sH^N3P?ZyJeps9&X zI|@j;kR#|(l4El)rQs?ZDy% z`W%B85@si*cN&@^wLhW_(Z3d?QUw+bcx~y`?dFsC1M~sjYgyp}WYu_xR zGT>%(0UQe1VRtz8kjrwyk@>Xs_8Zp8u_m(hs`u`E<%Q$Q0$#RBk zHPAPJ?iJDHtN}Yo+47TYa0^g=v&t;fer>khe#1e=uGUI$XriJb)jeV^kCN}e7?8cT zFG~y!f6k0UA-&+ka57O5>#<(R6PLs;zVbl(?h>_1i!OL9%UGcipire5e;RJypPrk4 z!dhDg|2?)VU>?o<>5b(M0JHm6=Lq(WFHVXw&X#!7Nqlgg3Ie3;(T1PkwqtTvSQczW zEC4&h0TB3u`6PRq^-t7vxsv5JA?y5-JVzWt6m1k*V%ip~J$GNe&EiDY|B0#AJ$Co< zdY>c{YiqUETkM^Er-tWcy<0U(<qDJeVc*$ z)3JVB#b@R-7t3pvpGILwVfB%y|!=6Zd_)Dn9uQ05n*w6k{zPNXa* zmruHNZtmIp%zzfwwmr^Rt7TQTwAVTKaI|XPD0h<)7Y-I2)9ZeX2AhEwW<{D~tY>E~ zQnp!(_XG}8E^|IOiN6t@2L;5QUD-;-L#5j6nbD%38Q-I`7cX+jSg?J-cxTxZbqTrxoO)kJN5H!JoIr?%rM1C-RP`Nu&ZJyot$n!@Ob)GV z8W-=71vkBRwwsjN&?xWd3?a0CEj|A<=lD^Y4?Dm|31gG0vGmQf?@SK}I&U>wz+`5h zyd*zFpWh3?>qV#^$K~O26}?o?H<8EOZlJ>ztD?!6DsOSU}sZniIpBA9vMJEl%rGQ_J|D)#XakkE#`_y4= ziu{5)_nQ8HPF{D)@bTYJ>ix&i>rQyKtYtuO@V~A(zqMGsEy+{Cgb;J3%kG`-efji_ znl#bp8`Ad~eDc8iHQia2WD7ldk^MFFXBHt)QR) z4xb+#$rJaWMR~=ypGtAr$)-(su90VHkA+Hhx* z%qntpzkc!SRr|_Z>kF%TIQnf;<{G*;3n}9Y8y0I{JGkiGJI}#af4vHR5arE2KvB4w znJqsL9BYtwdD`#jqD=y_h~;aY`<=tkz3buvG!`ds&qo263b zeWn+VE?T$%x7x{p!yKnM{qCpix7#l1en{1*o9Kv;8!T+s$IUj^WtT6HWdSvojQ^mi zk6Wj7ccz$2D7#Xgmfp`oe+@+dlCxQ#zB}JPX7sL{c51&eessk2j?OPvjiUQ5o%FDx zW^mb{Pgc*bj>Ljv^L#C0dYxt+#@FAyuKINPUP%mldLi}2e%i7{YG@^D(CMU z2oQlR^8HDK`2c!)0)=%SJz9+lF%3L|U)BuM8k#R@$Fk3%YA_tN@q0f}R6zG-A06GA z=5t|IW>%K4er2Dl*ntbqtu02=&-qnTRW?JM_^`y!UCWvYoE)w*s9(Qpl*_A$$W~k< z!tZNZds{tqU&#?8f-76s4J+(cgwT*AEnBs!%RMP|8RgJ+usI@8?_}6;rPFJkIC0_y z%q~QHy~6V=_ovj>bfPx?o9Q>gnuFeIc z26o^rG&i)XS|Bp6<#|@x#f6kaFhAYZ-{$$!Z%Y~^3L`=po5R5_%8{UM?%#T8@|;&67j zAZ=XRTr`sEP0yp=y}JV9B5@2e4S7pUkGWtoo3SDi z4F$NSj~dZvPlv7WJ(}fRaC{4w52z}?Mp9ESUf>IDCHx4n2{HcCp+6bMm@bez(I9Y6 zyjYX0Qr?l0hAX)U+(+_o0F8yPU00IdOmpVREr>?}DPOW@#iX0z+f({QqAE|3&K$nc zFKHpb1}hlf`6<#s;9^6cJ-!gLpJ_D>0WWW~7+-}5XwsoCnev?IqxX`MdmK)378=>I z69#lMI1lXd!H1mX`bTFQ@ zqC#tPfpOqV4v)lyeEK!hKc@N~ZU)7cwlfxK3CqWB@uV!H`uobfwlK)pJuO7)adM&3 z4RJ+J36Z6HqJg{)a#WXE7X?Xw*EJCtS3M?%5b`d`6#z|t0q}l5qi26C87mq+hA0WB zif`dvb$mK0df|5XAVp25RGH3!xZKEdbLIEw`_nB6{1IX0;!S730#XT3XZaaW`^Z-iBZPb!!vWdyaaITJBZMbboR;^RaJVs-Qt6#HCeJRooH*K z6<>e&Fa@yrbx8?kM{>8at3|BLaiFStpHR?r~j*j-$>icJghfw%w%8?TYxxgvLEW|iJG0W%z~1$oH!ktJrr+Zz-BHH z9iUR94Dj>=OBinFJzuDhek3F7tm_v%YV|fRFD_bIpj~LZqfVa0RI$+jA&{Cg78;Q0 z$-IdZqu0I_`sqQ0KoUaU>}=j;DriU~Fj^r>9AdhCEH%PujZ}}_jbtp0cY~=Qw?T4!1)zc$`g8-K|@8w&*q;wE+~#2R?-SxVri0 zF~EvqM`pRnN&knmry_X$_N@`)B2*sF>E2TKG7=%~vj~%go z-)#nayw{8wcO9-yGHBDP)kF@%2--Leu~a>XwBQIebPw7g`LMV;qTU8^BsFVl1z~M5(vdvxekMCT|GuDh*pCw+LhgUKfu;)3b#WT zMb)rMw139WIP}!<=OR(*PG@d&dVz=eT+N)9eBaATUuR~26URC9CI;)+^{%nIPj<`6 z%d@|enGUJg$^k~m|IneKVpz#4-Aa>!Yns0)*^s%!mf&v;vy3-8CFsbV!tcu$L50?oh;j=Qb}3sSlahB z$V=~GU?8nwH8<6w`SbPIbqnJm1anwn3HKVbF)Nuc$I$SvX&+bv$d$zgtNm;w=G~4% z<}0{a*;yQmfsP<7i`8l=@nin-;GliLHP7I?o;|lK?y=khOYmWwoIYco3+sZD=JuTZ zUS8c514hTF4;JqVk?ow=29|1z2=|bYd&V6RPs+EkcFwF>8CAy)9Wp|REwUH`1Auv2 zuCzG;c~jnYfY?#zYd$3Yk}#gL=D~;m{<~L1U7c}w%Ttd(1+|0Z2u2f zZ@Z$_bQd>ozH`_8liClOGyje=>54eC0*hjT*PvyVLxvR?W9c4Kcy`dwsD%wn1&^Sl zx~uc^!Hn^z8?0u%_TX*KJj`w8i8nM{VrsVFH|(KU z$9eGfi7X|K*5090ZR=cXs^aAZhGQZD4C>|U#<KneqUOua@d^Fxdfb| zY;vd+T53p*jhR`9c><3&jJt_J4R-P9l*Gs1dK%Gavz*}8&BC7T>pXfQg?=Pg96uAn zuh$qnxOTo1HNj#Byt5=%7|kni3Hrh+JwSEFL9O*umH`wIUujFL*-u+^9PWkdJ-Yo` zIWu_pi}xuj!cQLMW<21fQEfh;odsHwI`g}ZZK=BX=hF?wI_jakoT_5#^yz)n)wv>@ z?f`lfV-N>$)(NWoe$C)jJ9jA^bc*cCIOvsw5)fa#zj?jj5B+~{d>wvjk+KzHWzYqLiKjdUKh zN+G3Ee0=)K`QsG|;=2f_He^35p}k|lKHhy*_e zYCE=Lb%I@me!qR3K2SF%~wkA$C0h=5g8$#KE_-teDV#!!#(Bp{)e*-H>Q$Mq#=u8=`!|gED zIWDSsaqewW5fw4Pnga&ZH(l05af-BYp|3!XfqXd5XHJ{p+#887=nuf-cGfOHu#8AU z&bqb#PfF8MNl9s-uYt?z7q{x#_0C4aW?Ni>m406`vf^!)6gW~NA7A%s2-GykhQjiW zli8#M#vq*RGK`~5RK}CT&HFUl;!vUSU^eZ?YJxLK$rH;cbeLykWZ$wRo6YC7`1`=z zflg0wPp*RU%zfP#WGs zKHN0gayLXqYHI5Cibnd%F&XE>shK5>5h*NCrh%cM9k=GxHCF#_!NTl2GMAQvjuV$LzRY^wsWqGD*Umjbm+sh>3>LOk}P}z1HC9+Us733 zjqKAeG=KXA4zQOj*^}381VN086Es8Q0xDmqjk?hLY(tbihu~M*Bl)JkG6N!SW_>^{ z=D5|NkkbPX_i7{qrudfqBO+_ZH=Ocg#g(g>KzHQ5sdnh#g?y#1f^3-dIU8UxXv>8C zpz_#$_nl0KGk)>%+uvviNXrV#N@UUL8BCr$Icmhn`!sV@g=oQzuQgEU9;=;hTbHDg zHnwgRy=8kSF5vhJE{oAjN~#x>3KMn%tL|u92C2^R8@(QSsbn6rBU#n#OU=Bbrud+A z;!B2(9J*-ya1Kso+_1PAfUXcUr%{TAFO$BQ8@(;Mg&RIS^zTvM*!aUPn0`8{@!X|- zOoJB}-zxz{$+ax(wZ6jD|3h48Xk7`z%Fe#Yks^`$jMv(>>qxiIy=|S3mXfhYa#HE` z!r3NKrMF|cr0A4ec%T)Oh7ROtJI4;d{NL4;NrgXeH#kFv>B&*Q<5#zj8hw@3i!8Du zi7$rq_9zdvxTndR&YLn=0-LG4rSq1)6>C*qgB<0{vI>>hWhLo(WLJj<4! zmd&OF8UnMh_`o^!MXlR&bZI%}50I>a52K<>lLq{95pf?Lghg# zRWx+864G4a92>R_@yWq_WtX6X!Sy7&p3=+5rFbosn1~m0Y#}{GK6>;H(Lc#Q_;fG0 zy+*l|_R@4W99r`J{fpa!Q+7~K$GRoDj_TaJ$UqU;?6du?(0zgf`yM-Xw{!ZfTSJ{U zkuRH+kW`!#QEt~zxs194>Q2J25E3e)TnpEsJD>TFOvHd0(|kd6Stpcwv-e(Xpb=n) z^m~EZzaV(caF|-rJuu4ftWP!}TzTmo)7Qo1LX+DmyCzCGxqU zA#X5UZU})@Favd>Tvb9Dz`ZbtFfptPMEyK%1nyscUL{T9tIs5769 zBZ4P*+boR)hpXBjKGFV2L_O`gTh+HoRqTBTh2Qg?=2++^mW|u^T^jj6a~XuZ)Ix7TFk(uTmyV(0q{`B?P& z9gZQ~Fr=tnJA-UpNx6rr5IjZNdY<91EwN5xz_5iKeg3=~2SXajrDNx&8fR8IFKVb! zeZpam364x?-XVl?Dg28X;X3>u>j76=_TR7P{_agxITzDE?}vpK7M>;G|MNMi6^M@4Tf&R&(`Rci8&Vm>n{PaFSoc_l$K>EQAu; zED#RDEjXBa7-|LOrf(JV3;g&hRvh@O8ZRYmGp|3v;1?g=ZvOl&v@Jz6hv;+UBdlU* z0SVeNZ;Vj|+Tct(&;`8$w)p$|dsX;tnRB~#ZJEQ+OUcPE0a;|pQLal$AW`vGH%F-? zeEVN5!0%8&W(44A){I)DF5wB39oL>c>&kYQSrMH0tT`1hI)(oob5m0V#kc6g{**ka{0JbUC4K;L>~WhMMJ5GzB4>{H?F(+d>7fR?O8)l z2%Q#^cFD2LI~fGgo%jS!OCn?>uMf&kh^|%1bHXpKU$trrMYkZ1QeV-a$#85G(voh1 zyru!1^$niNSN=0FFz_1mH!@C%izF%s_^<^WqufUfI2tl^=ndeB=2XPU9~d6u>|Ws3uZDN|=%>dt+~D zQ?yXw9EVHFdo>ra1R9m5@Z|!4u_{H6A@u+Vx6OyHin4MyE_!SwtIFP*_-pk=B|`6p zvBZ<%ejt*Tm9z!KX@O{pEQN+fri5^HQT~f`qHEWE$oFJY4G&1h;hNgmsPJn1yqrr+ z0$K2CE&@-AH%U>%ymjN&t%$?%gh5#Ad;|*w{yVkoK_BcJQZ{h@Fvm2F3v2a*^2`Cb1q1}3(F|uSMtTpxd4%th4ZO*t+nt#xjh2h2}>0X3vX%aCxd=%5^wcj zlM#z<_jGWlMF4fGbW_4B+y`28*cjKidEW4k2@`4JN%}!tz(3XrZ(&uhy;;LX#j1+v z?cc+b>RRqQMcw1;=eM12QqBQ|Mz_c;qsxFi=qnfOV?@KS4f>Ge=|vMztMqHL0CZ6{ zCdcfxm#ZWMb^rrfNIKq1z2fCHWw75~FF}%Fuw^2a@w3v5+c$a9m#cVbwb6?bab;kR zC8KvpN=pWT4~y(g;eoIPR()~ln^mJE9aYK$93RK=2%zkN!Mw#0CaTD{&L;S)Q7rD^ zRukY|`y4S!e67GTB6!u@%+kPz3dgz?t;&!DCvtZFW0GeLSX`)tqvwNw3M2$W%p8x6 zz1{R4Og$TR{-Q-Urw;a?>i9H5@$6C;js&BSf4~nRXL;kSk<3Sv^D3u>o3rh^v+2Mz zPF75aSvRnaip0)SGBIu@;Qn5k?Cr^dL< zV#|5+_R;To+%C>AJ=h1sD=iZ^wdDn4ucoJGWnV4~D6m2$#x^>5XWU;J?^3`9uanD8 z;hsHzhVK_eZTVnwP}R5bDqddS#*-m5vkFEY&s6p}w5e}?@BXf$Kqe=%x)_<^jo<;4 zew%j7wkQ!trM_G>Yub%|Ih|Cc#Ftnw?jsacsX2z5&z*MF1>4(Ipxiv;$0DX%*QH{C zvXPihm^ZWzaqC)G-bsk*xKI(bXLMp&;_yVBhpz(?XZ2y7wNVM$mlAU&(#1;y7%n`90%;AroD?5A9m2u|jSCyHA ze*2cQJOOapu&+=4X5(Y)dkdxG$R{uUK^24eX;1Fj_v(`iIA26Wcqk#`1~Ck~z5+{WhC)B9vOE z$uTp$YyTQ^@9VeF<#+ZpXja>RLyYAUQ*Oa6n+nM{yxxVMde9yac8s)${~l(lbUiTR zm6uS2bQap2`kufj80Z%=*7sPpZXj0OKx{lR{PeEw@20$@xDv5|1xVNeFJSVi%=uo+N!mZj7-zQ48Jt~x^+Tb?cBZG(&}VN z&@M)oxqUhIx_i6^+`L$%FI>rqgs_`HH;7l^-_hp$X|_q6=CP{o)BjlLVd`H#&*(e4@BjJF>A?+FlasfD?@ zOaP~0Z_qy_q$J^G^}Tzi!$N*_DEgK17r#>c1Vm0X8>r$9nOrFyG~O+lzJYpO52QpHyr2gi=c^_){&S?Zaavd5;yE8v0cT>5tK^xdACcVGvm zaY^J(bzHg9wq#t^vysrma=GHPR4BM8EiP;~*sO~7k?f?mf3t{GtV(M;NX10@Mb({( z|1sK+n{2u3@^nYUQVSSW%D9#J!QF<;*Gt;_(s82hW~W7_N_Uq|o7+3|d>FYi^kUw* zE{^(}Z|*C+dH14Jy;WlLIZZ9`M*@(3;hbyJ)l~~ad2=!lE$Od-=k1kFH>u#I6AM`L z%%jq0+FaZCmDQRPD~1+N1qsO7^mtF@=GN$3+qVzc*-qbyYq6!8nst#%WqGEA6LMrW z=>K~zrQmdZ@8do{LxjRfv8eo65dYSMR%#g8BJUbsk4WT(04d05S z2H@kZ($x_PC7diQ48o^tfDJxeHu%v=jcbPIX9DLA4DUYv_D?Q3h@gQuBK zu`--uXt`!rOFzqazu(Rdfp#r#z45;8(_)y1-Ql1Px12lp^*C=gyxHNYyRRC)S2yf` zfA7R~_sgF=OBr7oTs`jHx)pZ1i*=7xyN(@u_My$DxStyn_{aIjyiIZF9*N44VQnJa zk?MxFiO(W6n}~(EeqZm4jEat)ps!z#;vJobi#&Qpez5hp%#G&G?Sij!A0mV(%QYB9 zscd#a#LY)utnmhB4J%(8qD>oM8;5m~$a_kmY?n10A)=2r?PuWRG>{y}$FElXK%daZ zi3SCb_vTGOspb+@8qg@G9WoM?Fvny_^gyk%{G8B{ETu zy2%JqV$a^avvbc=Zlj)~4U^eP!Vd`i)jzJiBK%WaQ|Axh6C!dCJ$0%j7xMq4hR;Vh z%PQFWA!cu~fIBt8Pld{nYY9%HcyU59Iv2~QwN zD4otvPB7k?pKp6ESmDyiY58)?!oeq&DB#3>U8cG-4ToYHyNRjh(~LF1cu;q*DCFg{ zuD!J!Ap%-Y0lIP@KJ57)U?GxxP_B`Rf2-dCMjB9q(JLZJV$9thP&+2>OkU_p3oM-& zQ^iQYkWhW}Rb~YVZA&5^4Jv%J@8bHIxp|MjciG}l+uvKm)`n$^(ni$B5@bMo=o=K& z7=@Cc2dot)trR>O4)RMrv>p$bUwxYSL{RVb^vY(olNb+T0=yz3`bhi6^V#h5TNx0E zwum>x_#hk=-u&AF$SX`Ve=;Yyx?ziYl61%wk{wdQ{}oLvX$MH5Rj5JW%Nh!5MwXB) z7T21JcM!)rz<(s{kR~Xj{F6*Y^B#Ql8dgwGV6*7&{xKg-tS^iKqAuFDYl6gO!n@@W4=-;Am!;5RSx6s|fE78VD zkPVeUR$PVULr?9@-rf=4!dO}ZT|ZvBmohb~Eh+6jVDzNr{v*k4)O&U7)%$>?3KQ(y zxh53TG8URYJ{la{fB5i&$?jNkj2@ncj^sngmt_L!)yUdeW#$J;Mln-`bp zKK2p+GHjTiWOI`S)bR~LNPQbNY*_mJ`w0{V4|oL$+Xf1lYG9yLDsvgG5%EMwRA5kDO>gy3N-p@zFfmLmWR+0sWfWTeXE{4N~*^xlO}oI z`DffT0|UH}I7ItoW}abqf=bh-1Q60TyjqhiPg7?VS%BYT7A-ZY4!4{+^V}&j=jw^Y zEp*ns*Q3%t=<7QHe(Rpi$V9TW9sr~TmCVqs*x}_x=fj6Rf&s(JD=H*pD+Ey^1?Fy& z(K*61`F~MY>@xThT`a^S>)xQeyu4!(JhX)68y9f5hQd?(SKC-WZS&aB@6%F;C5F!; zzH~SLJl@{y-880;PLYPT&T;vu+=`NY&WUb>B+IxHiRBMAeYr9Tas3JAEJQqocrr2g zS9b$dm-z14`EIq<8*G@js~UaY)_Q!rSIk&i#l0Q0wjtEYKG6v!oQ#9_@)AV|%Uz=X zbr=yT4;f8aMP+3UbgzgOUKbVVgD3F4Aq$|AZO5ZGr>NuIJ zxK!Q&a)F%ZQHdj!FF_U~ftGK7m^k+mV5%1}B{BJn7nv>YX%#=G2M{7D4XsHD(7IX$ zbKY0oTxUJ}bKJUaHMXMfZoh_51N=L%6&0tV63w28^2TudVIx zb+}72&(t1Fg}gZ<{5K#&f_vt@cMf3*Qqeo()Du3`jD_}id#Tc zH7_@J$ZE6>$KivKRhgP|&e9G5*KzY^hS z6VmMMUoGmZb$F50!_Nj;&KiaT+WFzPWUXg|XbVMD zOw8_G;8%T-9B|(dvQMVsF}KS8tp_`e{LpLAe>oAavo5%lMI8N@_0j5Suj$HvFn06; z$V>i;q03J(T5-6}uy?eh%cCVm`4v5layvdg)I`i|`1S6v4Xl4ptfx~;1AV*}cS zT~P(y2%(GlK9cJxxXZkY>!wU^f4Fd9zkZYB{_PjK^v@E8tJqC%495I|5d{8&*wxsr-|#lJdH zPS7TS+-7*RAmNC5F*}JH9rQ=*@2~Td%R3fkJitQIXEVUyf4N~K2yQ_HDwAWy(FyH? z0ZBmJxbGTA5;zR-1l)V{sL}C?uPL7vC%9iMqR8gJ-6klNZK?8uesOjq>b?Ps%{m;o zE@JP;oHfc1lOAXdXPF5K4n5YZ{dASzPMZ1uVoFBZ8_1PVHrb7Ltnolo|8b?`M2i#{ z0I^C{{`lk#Uvc?|nG%#Fl@>&@ZlEO>A|n!^B#nU4Y?)?Ccl}>Cyc&%O5ZtWTU&X}^ zWX`<+)Fz5e9rPN2ZistL)y$uc_G@C!bTRpKvf0YwF_s~j0aHsfHvNOOp*OgXN#&UQyKVYsK&ZHNgQq$E zzA$1FI{d@opQg|K43XW8nBV6`-`Nn5PVlnZC6Z;2JUx}%91V#LUt4U z;7uvQlT?y};u0_Ccixm*^i2|$k*y@JVnHV-En|j-WPNLqnCM#dEnUL@NQ0kQ8Ub;>nW(6xXO$u&PH5 zrQRK|VPMNfxmk4<|pu7x7W*Xc#&N4h?H3yr(aw4PlK zfi*NX^)7?#!Cea0C<#T}o|QHA?M6@o znA{U{4g;49@UX^(i%A`Tm-#`dWZU4hm^T}c5RQSWO_JD&s}N`7nlnz;E_E_8PY$xB zZ6r|i?Ud;>Q}p&WCmyaFIQ#1Ig2P!4bDx>#hOb?>&H#nNDY}V>&INtLB>8dE!GM4w znjpm%DiXp>ZKn^iUjx>ML{XKN_0Vs|raip`C9w_go^=0JG`0RN%VNTGsuf&CX^>^~ zcxl9=1=_w%sVZ$k3z3%}M^=Ol;r(zbgDei=kfjAJBd|ouj&VXawrWq2TKb|k#q^} z%WII8%W85G!)x~=f1k9kv+t(Dzb*$=$7HRg5TXAO)p6_f&^c(ZwPyGyEgPR!!f6;r zAFWq5KLyqvLM19&KRCtAer54TQ!RI=bUnkfU51@aw$<7@;v!f1xM5~YYljWFFsJL` z%D){;{)*~4zD?PtfJ&>~`Ge=}TFPlxe~WI#>(*JBnLYlDAhFXQLm*j7&>IgaZbt{YHP{cx12qD8dZ4@L{X(E^%F|81k+Z- zqE4SS3m+>9MlcKaVBUk$DmmlpZ;|Fp>OV9R)ZQ(n3xR=UQqrvGY=-uQu#E0ediK-N zIV@wC@X39L&K|Z*XZ^>?eC{wv;vCvvnW&`u>qWh(lP7m)B8aUBY++|vsu6*HMlq;w(*@acwBC%mSQ<_4@ysH%D-pCamz!BU}6F!!k1`}OLzLz)Vn zc_eZqbMWR&a;;<dkynEQ^ioKv2S+X|j(6J^?pX?X zB;^RCFWCH6elOY`mF4Fp)5R7!Ihnv_f9LYfAx0+Z?Ae1fVPvcwAU%kFMrMS;!<&oj zmoY`)uto;|mc(<+!VB+>iH(&$A0Q($EzJXjX&UYtyIWJ#7c37&4|tR;iEn3i6a~q4gIK4bjq4yFv0QwqFd2aH}}-%1h&twW-OQmoNxa= zeT?2bikte=Tox5RM^s|OWLr~W8o?H!$%NF4{*+34)$xtFN8&gA(}Po{Rcmh7KT&-! zAv(G-ckITPf>R@yM zzX&Q{vv==am;>y^TTS~}Zaj97Sp;I{7hX2&?wvbE3m5jJ@J-t>%{>i2Cl+220-zog zOCsOi8Qp0?W6j6${{G_~w|I4L=RLuIic9c!_;f-RnlG?H(kK=;5!_NYSv3 z&U$vxBevrM%cLWgT}?)WdF2|N_~0^DA|cMWS;8D+3GdY^v%(?m^lbmn;g(9}OUfIP z3+(2_O|8CiQP=g`E5p5OqHR)6F0V5EcpJU{7t7j#_Z(uY2|pVd zs7mjzz#=&~2`ZZR+PKM;XvcMw_~#Et&Ke-YA%Ls>n(SvvgI(&*WqaJMsvbPHuI%+W zC5X!!yMo)p1y$)~8$P9e+3o(kpyhx88r?mTv=@u!PHc5TR&G>J(~X9Pvc^*LEm)-j4gQyZYj%xh2bEQ*$=TCw|^|%&vu$RPw)yhBxB> z&wrGz47y3cU(J}^E=zdkAxuGbIt;xR5)tt>?vN&P>fvC{AAUF9AB4%jlfOpT#fu$5 zU%ps2K%&vE-Jy{$fj6F_9;6aUXIa-|EqK-+5&}XUdDDfQUnqD7%s$tY{Xk6eo!Oc{ zx#v3Iv{<`ip9U<4=J`uC?j7^;@+3%PzN4czxY<(YyTGt^D_8Dk0U6r!$UBl4c`0rB zmzC2RwEHAEuE1F}KWwX$Zp|aDXV$ZJaffdrBwVL%w04Q4{>eW3oa1lHIy& zl4s>&VWYjv&uv6JaVWIEvXIaj&@hR%hdp$kfE}4j^JYsQ9u(n%OdVTD5vVwPtR3}A zb54{%gwR|cT(u3z7J11B57cqi5G)@yTr#5Z&n+lruBHLgK-TJA((yb-qz7W>4jnr# z`0=p-`ltRq*X~ZRAJ9%iD6pC$J=QBz)xyAKxOww9sp%AYXuU-`K$~-ehva}?> z*-w{%4&cbG^d+*-sT?JegVl~=o&+8}9;_;&D)4aLFlxmaZ?5d7nC=1Z0-Ej#j9&Ar zs0Cs&=47`3IY*Az*L3b&8SNx;BgGbaDIv&^IPo0y7}&&-yORo00?RUI_iOXVD2@|S zQhTrpTdbH7L%O!8?8x3jPPOk=Tyu~-5sZTe$N;|Vd<9fLHq2C}ZMP6(!oNR+&_Xkm z3PeKvVR<-3N9|z2#aRQ4D14+>uXuN?2!?~VaEj7z`kE%KAr&aq0bZeYU9t( z|8`z(%jaje7I9~Z4V4R&`uEyaN;FqO|7!P+vzgKKh3H<;$%kP}S?UP7)cY92csx25i}83qQ- zdX_&eL_VgpvTAKsW~N3bCH)b0a+zR=fJ8uFYF3;}#<*FI^TY)kMV1Fx$#>F)JuH_mTI1;5J}3s4>&rNy0pEAk^}z)c7l_$z7t%&lJmHcf*y@ow?GY4`Z7n?ZzZmOaxJ_eEI;sHvv~I1>Vtv{;5}J?U9?59(%BP7_w=N3%N-h9z-jk z$ikp0N88X5`ukmg(SvvLdoudCy6@yqy#|dLbA&~^_9^YdMLgtxX36>>Fzg#p!{Kc3 z-Q91Lva*MIM^68qvpEFt`ewb!!!4@A(0EntV-Jwzy7w8>qsLZ(2lB4$mdk^CH>gBB zi$rxW*+|{tToZ-mtXVL@ocRZjkkKo_e?d|sBqk+=Q_J+x)zH`-WWYy1L-=A^kza%F5cs%L_VSSic0U-!fnGrqi>_K^(n310r?Qh8Jw4kEnB;i;_9V z?zL`z)}p=pYpDtprWAjws+?NB=&p}`qM8)ragZY(TJW9kfTXuSAb9cewGIwOUwe#1 z)f*W>0RfZXAtg5(uyf%$3tQW}N4+5n`i z+qb@1 zNkz_&nNG`&$`M?25)}X(T^$9YRDw)ttP-9~Qb1B5kNkzVT@?M&d*WJkaYAM*s?n4E zA3S_0YFSgI@-GGQ{p}>-Bbp(SzA^Z5^20sma{r3dAe;V6n;uVQsY@!s=N$^@Gww`@ z>R@k8wQUkR`#0zUK87aAVPRm}HsdZ{xFGQuU#CNLYB3`awzwxOQS2uF4^f;I*>rJN zR^v=C>iWU)5@Am5)@b;RL$eF&B+bu_HnzT*hu(euypIC5zrTrc<8hZu9cIsd-0lrt zobH;M-pTQCamvcWbNrhYKGNBro=PCULIIz;)c5C+!qW>jDdx{hJIB{1B>!->aYiWLgVpLEuMM4UvpbCgZ|{ z(cwT@*nZs`(C#Flsc_&X_pT>vqOjD+?*c`l;@`(hGjvQ?9g9CTy!Y{mq+0+^fugpw z-CW*QF=UeiSVv=4=z|zz>q(=@Y7kb2o8TH5EEr0>!G`7owXOM?o1Y(g`SKikanJW_ zlIMtF0q=j~o*ti?wMnp#j)`&jrX3YAxP-mma8Q6Z9}Ur*IueflVJA--M}Lel9eJHj zeFwGx?!LfN*RJWkT}jU?nYE%bM{dk=3Thk`u}M4R*J{t0{o@unYyRFqA5W=jYHOR2 zr>eQ~x4tLbuuRJ;j0C8oxxAI4K-e)uVS?<2Jw{!X!2Q}U8XI0Rx|KX3l zP(X7DrNg{QNCkb7@DEh%6J1@$ic*ISPov9hm#Oic)hCjY=Y8?;kt4Au`yLM-n2|3M zESaIhnndSedmMbo*7ci5{^O%ddoJN#;xzX7pAbnpEyi7`k&2t7v7K7zufPA@{HkN8 zPIVL<+dH@mNSk=`iqYfysaxwVUBA>|2MQe-vz1E0OAM~KV~dKy);*XDXGpP?)CdtCP>;uxL;bk~Al4=FMxjTZDonqz%zy^u%wLe$ z<4RY0G^Aj!*R@hp>)-dhI@SA44rn(1H^QvOx|HtN)oJLWMjXOv?=Ndtni&_peY=x; zdhFj{^`bi~{tT?q-P}>$rxc~DoIWf29_eEpKI8i$=Pup44emRJu;JwMM-r>@nq7>` z3+8kH5c@X1wrsrkGp-V%-mYG}j@rD3`SvcY$IQ&{y1QJt9(F1)1*}t*fS13Tw2L}8 zJ2;rY*H-W-fm}~ADaWmPBfhhj^dc7s3XowvpbAHc*ZPp2Sh03gX9A`tN2mC@X(dec zj63bYTNZg5&}p4}Pu}^5Jn9=>!E>Tc5zR0R!V4BCy`UZyfmb6cQx9;u{)0KBybsPC zw~HcTJ6=^u(80O1fP^9OGt`tPxQDpc)-aEw1L-BZMJ~sPB%vD+fTNtpWuryngzRD9 zUfyd9@dvtAya^Di5)H*mjvVrSkZv63B$J3aN_Jmpg3C2^w!qqn=@q&ZMX?+RE1q3$ zeB7mWxS=F(313a?CV@!Hmi1@kYhy5Qp$ZZA%1A0PYl#sHVZcOj4J*Xhg9c8NdL;J< zraDARYXYN8SztR66Na5V+XiC~B}9Mm0+E~e5;@m3;;HZD=I-FKkzslSS!}_NjWp`b z>#}d(uJd;pJKP+1wS>E(O5>KeArY*o)H(P?4Qx;!sFBiXZ>7tRk3mP5({G5x5|jNt zHoKg6jP>SC>p_pz8NTwvJu>XrjG5!Ka^BACPZci^r}(|`$#5wqnKA%9} zNPetGYcHYVqzh|N(3o0THDp8&O7J2sWkCcXNYFib@Jn7@`T0zrUPrg`eNV98iQQX@ zXNlRvgTWmY3a)1vxkyCX>wj*E6^zOdY{p>9lwD6W3uz0<-|NmN1+J8D11G5XnW#3Q z--eyL4|f5K(TU2KDO@{^)GP}sIq!qq)}S@Jker-!%6H`HB2I)JgeOwA7*J_*GRqim z#lH!)n_XQ4r+la6`pUJ<1yM&q+P)Vih*X>t)MsK54EQS;_2UXI7+CZVaaR=E^sF_Qr%8q3@^?#U3U z%4jS4fM=V&j9VN!*V1wa6mGkOXr5|eXmfbx?T|L|y%BTQf_K^AURC)@AlmDH++sVS zNSGtnLR`XNe`ne|uW6+!@?Xs85@6_iF2BC;?L}SJ@gB&_WV_g zL-rNj5}v9BM`!b#IE(F;E+rST#o|qS2#lr0#ulaDuZ@@#h)WP?&fdQFfBwDIaC{Pn zO0^>r2m0q#hzlgf7KR^bT7k}M9yZj0J}Px)e|z#k@h!I^6XQAupE7>DCh7Hn7N~ng zf?-SwL&4EU4_&Fgd92aUhB2J!?`oWl8OCKa==cU=mmf-C3wQ!caL@f0lTe43p!t#C3Fkob3gt7J<7t9{aW$G3*~~a~xKz+U z_oR^H9LA-?pr|bziXgE*s9vEF*yPFO3lNtWZ(dX`>z$Wlv%FxfL5?<~AwUcGvybJR32Guic2!KI=)D*#?(MyXdM_66YBg0A?+&71Xk{2fNG=`~C(o_dkP7$r&fR=egeUVQ7(RgEzc z5lN=`ZnoidWcK3hr`9l2Dm{>Qy+n4$fh*xwfRlsYjmK?5?Y@n3L?W#vZx%#MR4!c7 zilw?g1|XNdU&JQAPpd6qY8<2h=Tz4|50)%nelkZ#mLCsVCZ)0Ja4VpQ zq+m_MhVuGHk8I*9A|oT8c=oEDB6HGU!R?MUIt|)XWny zQ~-`Y#JAan#(&V^lx`nOEZemk=z<9gqFKBne21}f&i|MZ2|6SibRIwVA&(!HDqzFF zcAQF0yz74KwU4t){`aO^qu)CI|NT4vkx^Ukczu%vN4^YOs^EW945u5!Pn^H){{cn5 BwZ;Gd literal 0 HcmV?d00001 diff --git a/docs/changelog.md b/docs/changelog.md index 5b6777a..a3c8ff5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,9 @@ All notable changes to aha will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +### 2026-03-18 +- add DeepSeek-OCR-2 + ### 2026-03-17 - add PaddleOCR-VL1.5 model - fix qwen3.5 position_ids create bug diff --git a/docs/changelog.zh-CN.md b/docs/changelog.zh-CN.md index d4bc504..4e29562 100644 --- a/docs/changelog.zh-CN.md +++ b/docs/changelog.zh-CN.md @@ -5,6 +5,9 @@ 格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)， 本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/spec/v2.0.0.html)。 +### 2026-03-18 +- 新增 DeepSeek-OCR-2 + ### 2026-03-17 - 新增 PaddleOCR-VL1.5 模型 - 修复 qwen3.5 position_ids 创建错误 diff --git a/docs/model-card.md b/docs/model-card.md index 13f7462..15e4167 100644 --- a/docs/model-card.md +++ b/docs/model-card.md @@ -7,6 +7,10 @@ "OCR:" | "Table Recognition:" | "Formula Recognition:" | "Chart Recognition:" | "Spotting:" | "Seal Recognition:" ## DeepSeekOCR +#### Main Prompts +1. document: \n<|grounding|>Convert the document to markdown. +2. without layouts: \nFree OCR. + #### Metadata * base_size: 512 | 640 | 1024 | 1280 * image_size: 512 | 640 | 1024 | 1280 @@ -25,6 +29,15 @@ | Large | 1280 | 1280 | false | | Gundam | 1024 | 640 | true | +## DeepSeekOCR2 +#### Main Prompts +1. document: \n<|grounding|>Convert the document to markdown. +2. without layouts: \nFree OCR. + +#### Metadata +* crop_mode: false | true + + ## GLM-OCR #### Prompt Limited ##### Document Parsing prompt： diff --git a/docs/model-card.zh-CN.md b/docs/model-card.zh-CN.md index fd4e062..4ec05a7 100644 --- a/docs/model-card.zh-CN.md +++ b/docs/model-card.zh-CN.md @@ -7,6 +7,10 @@ "OCR:" | "Table Recognition:" | "Formula Recognition:" | "Chart Recognition:" | "Spotting:" | "Seal Recognition:" ## DeepSeekOCR +#### 主要提示词 +1. 文档: \n<|grounding|>Convert the document to markdown. +2. 无布局信息: \nFree OCR. + #### Metadata * base_size: 512 | 640 | 1024 | 1280 * image_size: 512 | 640 | 1024 | 1280 @@ -25,6 +29,14 @@ | Large | 1280 | 1280 | false | | Gundam | 1024 | 640 | true | +## DeepSeekOCR2 +#### 主要提示词 +1. 文档: \n<|grounding|>Convert the document to markdown. +2. 无布局信息: \nFree OCR. + +#### Metadata +* crop_mode: false | true + ## GLM-OCR #### 限定提示词 ##### 文档解析： diff --git a/src/api/mod.rs b/src/api/mod.rs index b0fb7fd..73074b9 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -262,6 +262,7 @@ fn which_model_to_id(which_model: WhichModel) -> &'static str { WhichModel::Qwen3vl8B => "qwen3vl-8b", WhichModel::Qwen3vl32B => "qwen3vl-32b", WhichModel::DeepSeekOCR => "deepseek-ocr", + WhichModel::DeepSeekOCR2 => "deepseek-ocr2", WhichModel::HunyuanOCR => "hunyuan-ocr", WhichModel::PaddleOCRVL => "paddleocr-vl", WhichModel::PaddleOCRVL1_5 => "paddleocr-vl1.5", @@ -289,7 +290,7 @@ fn which_model_to_owner(which_model: WhichModel) -> &'static str { | WhichModel::Qwen3_5_2B | WhichModel::Qwen3_5_4B | WhichModel::Qwen3_5_9B => "Qwen", - WhichModel::DeepSeekOCR => "deepseek-ai", + WhichModel::DeepSeekOCR | WhichModel::DeepSeekOCR2 => "deepseek-ai", WhichModel::HunyuanOCR => "Tencent-Hunyuan", WhichModel::PaddleOCRVL | WhichModel::PaddleOCRVL1_5 => "PaddlePaddle", WhichModel::RMBG2_0 => "AI-ModelScope", diff --git a/src/main.rs b/src/main.rs index 9787ca6..99ae64e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -258,6 +258,7 @@ fn run_list(args: ListArgs) -> anyhow::Result<()> { WhichModel::Qwen3vl8B, WhichModel::Qwen3vl32B, WhichModel::DeepSeekOCR, + WhichModel::DeepSeekOCR2, WhichModel::HunyuanOCR, WhichModel::PaddleOCRVL, WhichModel::PaddleOCRVL1_5, @@ -520,6 +521,10 @@ fn run_run(args: RunArgs) -> anyhow::Result<()> { use aha::exec::deepseek_ocr::DeepSeekORExec; DeepSeekORExec::run(&input, output.as_deref(), &weight_path)?; } + WhichModel::DeepSeekOCR2 => { + use aha::exec::deepseek_ocr::DeepSeekORExec; + DeepSeekORExec::run(&input, output.as_deref(), &weight_path)?; + } WhichModel::HunyuanOCR => { use aha::exec::hunyuan_ocr::HunyuanORExec; HunyuanORExec::run(&input, output.as_deref(), &weight_path)?; diff --git a/src/models/deepseek_ocr/config.rs b/src/models/deepseek_ocr/config.rs index df9c777..72d5420 100644 --- a/src/models/deepseek_ocr/config.rs +++ b/src/models/deepseek_ocr/config.rs @@ -90,10 +90,17 @@ pub struct SamVitB { pub width: usize, } +#[derive(Debug, Clone, PartialEq, serde::Deserialize)] +pub struct Qwen2_0_5B { + dim: usize, +} + #[derive(Debug, Clone, PartialEq, serde::Deserialize)] pub struct Width { #[serde(rename = "clip-l-14-224")] - pub clip_l_14_224: ClipL14_224, + pub clip_l_14_224: Option, + #[serde(rename = "qwen2-0-5b")] + pub qwen2_0_5b: Option, pub sam_vit_b: SamVitB, } diff --git a/src/models/deepseek_ocr/generate.rs b/src/models/deepseek_ocr/generate.rs index 100cf4a..f435113 100644 --- a/src/models/deepseek_ocr/generate.rs +++ b/src/models/deepseek_ocr/generate.rs @@ -30,6 +30,7 @@ pub struct DeepseekOCRGenerateModel { device: Device, size: Vec, model_name: String, + version: usize, } impl DeepseekOCRGenerateModel { @@ -40,13 +41,23 @@ impl DeepseekOCRGenerateModel { let cfg_dtype = cfg.language_config.torch_dtype.clone(); let device = &get_device(device); let dtype = get_dtype(dtype, &cfg_dtype); - let processor = DeepseekOCRProcessor::new(device, dtype)?; + let model_name = std::path::Path::new(path) + .file_stem() // 获取文件名主干（不含扩展名） + .and_then(|s| s.to_str()) + .unwrap_or("deepseek-ocr"); + let version = if model_name.contains("2") || cfg.vision_config.width.qwen2_0_5b.is_some() { + 2usize + } else { + 1usize + }; + let processor = DeepseekOCRProcessor::new(device, dtype, version)?; let eos_token_id = cfg.eos_token_id; let bos_token_id = cfg.bos_token_id; let model_list = find_type_files(path, "safetensors")?; let vb = unsafe { VarBuilder::from_mmaped_safetensors(&model_list, dtype, device)? }; - let deepseekocr_model = DeepseekOCRModel::new(vb, cfg)?; + let deepseekocr_model = DeepseekOCRModel::new(vb, cfg, version)?; let size = vec![512u32, 640, 1024, 1280]; + Ok(Self { tokenizer, processor, @@ -55,7 +66,8 @@ impl DeepseekOCRGenerateModel { eos_token_id, device: device.clone(), size, - model_name: "deepseek-ocr".to_string(), + model_name: model_name.to_string(), + version, }) } } @@ -77,6 +89,8 @@ impl GenerateModel for DeepseekOCRGenerateModel { let crop_mode = extract_metadata_value::(&mes.metadata, "crop_mode").unwrap_or(false); let seed = mes.seed.unwrap_or(34562) as u64; let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed); + let base_size = if self.version == 2 { 1024 } else { base_size }; + let image_size = if self.version == 2 { 768 } else { image_size }; let (mut input_ids, images_ori, image_crop, images_seq_mask, images_spatial_crop_t) = self .processor .process_info(&mes, &self.tokenizer, base_size, image_size, crop_mode)?; @@ -152,6 +166,8 @@ impl GenerateModel for DeepseekOCRGenerateModel { let crop_mode = extract_metadata_value::(&mes.metadata, "crop_mode").unwrap_or(false); let seed = mes.seed.unwrap_or(34562) as u64; let mut logit_processor = get_logit_processor(mes.temperature, mes.top_p, None, seed); + let base_size = if self.version == 2 { 1024 } else { base_size }; + let image_size = if self.version == 2 { 768 } else { image_size }; let (mut input_ids, images_ori, image_crop, images_seq_mask, images_spatial_crop_t) = self .processor .process_info(&mes, &self.tokenizer, base_size, image_size, crop_mode)?; diff --git a/src/models/deepseek_ocr/model.rs b/src/models/deepseek_ocr/model.rs index 318fe4e..5a5de31 100644 --- a/src/models/deepseek_ocr/model.rs +++ b/src/models/deepseek_ocr/model.rs @@ -1,4 +1,6 @@ -use anyhow::Result; +use core::f32; + +use anyhow::{Result, anyhow}; use candle_core::{D, IndexOp, Tensor}; use candle_nn::{ Activation, Conv2d, Embedding, Init, LayerNorm, Linear, Module, RmsNorm, VarBuilder, embedding, @@ -15,12 +17,15 @@ use crate::{ get_layer_norm, }, deepseek_ocr::config::{DeepseekOCRConfig, DeepseekV2Config}, + qwen2::{Qwen2Config, Qwen2Decoder}, }, position_embed::rope::RoPE, - utils::interpolate::{interpolate_bicubic, interpolate_linear_1d}, - utils::tensor_utils::{ - index_select_2d, masked_scatter_dim0, nonzero, onehot, prepare_causal_attention_mask, - quick_gelu, topk, + utils::{ + interpolate::{interpolate_bicubic, interpolate_linear_1d}, + tensor_utils::{ + attn_masked_fill, index_select_2d, masked_scatter_dim0, nonzero, onehot, + prepare_causal_attention_mask, quick_gelu, topk, + }, }, }; @@ -417,6 +422,7 @@ impl ImageEncoderViT { // rel_pos_zero_init: bool, window_size: usize, global_attn_indexes: Vec, + version: usize, ) -> Result { let patch_embed = PatchEmbed::new( vb.pp("patch_embed"), @@ -463,7 +469,8 @@ impl ImageEncoderViT { let neck = Neck::new(vb.pp("neck"), embed_dim, out_chans)?; let net_2 = get_conv2d(vb.pp("net_2"), 256, 512, 3, 1, 2, 1, 1, false)?; - let net_3 = get_conv2d(vb.pp("net_3"), 512, 1024, 3, 1, 2, 1, 1, false)?; + let net_3_out_c = if version == 2 { 896 } else { 1024 }; + let net_3 = get_conv2d(vb.pp("net_3"), 512, net_3_out_c, 3, 1, 2, 1, 1, false)?; Ok(Self { // img_size, patch_embed, @@ -559,7 +566,6 @@ impl CLIPVisionEmbeddings { } fn get_abs_pos(&self, tgt_size: usize) -> Result { - // println!("self.pos_embeds: {:?}", self.pos_embeds); let abs_pos_new = self.pos_embeds.clone(); let (len, dim) = abs_pos_new.dims2()?; let src_size = ((len - 1) as f32).sqrt() as usize; @@ -1107,19 +1113,101 @@ impl DeepseekV2Model { } } +pub struct Qwen2Decoder2Encoder { + model: Qwen2Decoder, + query_768: Embedding, + query_1024: Embedding, +} + +impl Qwen2Decoder2Encoder { + pub fn new(vb: VarBuilder) -> Result { + let qwen2_config = Qwen2Config { + vocab_size: 151936, + hidden_size: 896, + intermediate_size: 4864, + num_hidden_layers: 24, + num_attention_heads: 14, + num_key_value_heads: 2, + max_position_embeddings: 131072, + sliding_window: 32768, + max_window_layers: 21, + tie_word_embeddings: true, + rope_theta: 1000000.0, + rms_norm_eps: 1e-06, + use_sliding_window: false, + hidden_act: Activation::Silu, + }; + let model = Qwen2Decoder::new(vb.pp("model.model"), &qwen2_config)?; + let query_768 = embedding(144, qwen2_config.hidden_size, vb.pp("query_768"))?; + let query_1024 = embedding(256, qwen2_config.hidden_size, vb.pp("query_1024"))?; + + Ok(Self { + model, + query_768, + query_1024, + }) + } + + pub fn forward(&self, xs: &Tensor) -> Result { + let xs = xs.flatten_from(2)?.transpose(1, 2)?; + let (bs, n_query, _) = xs.dims3()?; + let param_img = if n_query == 144 { + self.query_768.embeddings() + } else if n_query == 256 { + self.query_1024.embeddings() + } else { + return Err(anyhow!( + "Only support 144/256 seq_len, data {n_query} illigal" + )); + }; + let brach_query_imgs = param_img.unsqueeze(0)?.repeat((bs, 1, 1))?; + let x_combined = Tensor::cat(&[&xs, &brach_query_imgs], 1)?; + let token_type_ids = Tensor::cat( + &[ + Tensor::ones(n_query, candle_core::DType::U32, xs.device())?, + Tensor::zeros(n_query, candle_core::DType::U32, xs.device())?, + ], + 0, + )? + .unsqueeze(0)?; // (n_query*2, 1) + let mask_up = token_type_ids.repeat((n_query, 1))?; //(n_query, n_query*2) + let mask_down_1 = Tensor::ones((n_query, n_query), candle_core::DType::U32, xs.device())?; + let mask_down_2 = Tensor::tril2(n_query, candle_core::DType::U32, xs.device())?; + let mask_down = Tensor::cat(&[mask_down_1, mask_down_2], 1)?; + let mask = Tensor::cat(&[mask_up, mask_down], 0)?; + let on_true = mask + .zeros_like()? + .unsqueeze(0)? + .unsqueeze(0)? + .to_dtype(candle_core::DType::F32)?; // (1, 1, n_query*2, n_query*2) + let attn_mask = attn_masked_fill(&on_true, &mask, f32::NEG_INFINITY)?; + let xs = self + .model + .forward_no_cache(&x_combined, Some(&attn_mask), 0)?; + let xs = xs.narrow(1, n_query, n_query)?; + Ok(xs) + } +} + +pub enum VisionModel { + Vit(VitModel), // vb_name: vision_model + Qwen2(Qwen2Decoder2Encoder), // vb_name: qwen2_model.model. +} + pub struct DeepseekOCRModel { // config: DeepseekOCRConfig, sam_model: ImageEncoderViT, - vision_model: VitModel, + // vision_model: VitModel, + vision_model: VisionModel, projector: Linear, language_model: DeepseekV2Model, - image_newline: Tensor, + image_newline: Option, view_seperator: Tensor, lm_head: Linear, } impl DeepseekOCRModel { - pub fn new(vb: VarBuilder, config: DeepseekOCRConfig) -> Result { + pub fn new(vb: VarBuilder, config: DeepseekOCRConfig, version: usize) -> Result { let vb_m = vb.pp("model"); let sam_model = ImageEncoderViT::new( vb_m.pp("sam_model"), @@ -1143,24 +1231,34 @@ impl DeepseekOCRModel { .sam_vit_b .global_attn_indexes .clone(), + version, )?; - let vision_model = VitModel::new( - vb_m.pp("vision_model"), - 224, - 14, - 3, - 24, - 1024, - 16, - 4096, - 1e-5, - )?; + let (vision_model, image_newline) = if version == 2 { + // v2 + let qwen2 = Qwen2Decoder2Encoder::new(vb_m.pp("qwen2_model"))?; + (VisionModel::Qwen2(qwen2), None) + } else { + let vision_model = VitModel::new( + vb_m.pp("vision_model"), + 224, + 14, + 3, + 24, + 1024, + 16, + 4096, + 1e-5, + )?; + let image_newline = vb_m.get_with_hints(1280, "image_newline", Init::Const(0.))?; + (VisionModel::Vit(vision_model), Some(image_newline)) + }; + let projector = linear( config.projector_config.input_dim, config.projector_config.n_embed, vb_m.pp("projector.layers"), )?; - let image_newline = vb_m.get_with_hints(1280, "image_newline", Init::Const(0.))?; + let view_seperator = vb_m.get_with_hints(1280, "view_seperator", Init::Const(0.))?; let language_model = DeepseekV2Model::new(vb_m, config.language_config.clone())?; let lm_head = linear_no_bias(config.hidden_size, config.vocab_size, vb.pp("lm_head"))?; @@ -1210,41 +1308,88 @@ impl DeepseekOCRModel { let image_crop_i = image_crop.i(last_crop_num..last_crop_num + crop_num)?; last_crop_num += crop_num; let local_feature_1 = self.sam_model.forward(&image_crop_i)?; - let local_feature_2 = self - .vision_model - .forward(&image_crop_i, Some(&local_feature_1))?; - let local_feature_1 = local_feature_1.flatten(2, 3)?.permute((0, 2, 1))?; - let local_feature_2 = local_feature_2.i((.., 1..))?; - let local_features = - Tensor::cat(&[local_feature_2, local_feature_1], D::Minus1)? - .contiguous()?; + let local_features = match &self.vision_model { + VisionModel::Vit(vit) => { + let local_feature_2 = + vit.forward(&image_crop_i, Some(&local_feature_1))?; + let local_feature_1 = + local_feature_1.flatten(2, 3)?.permute((0, 2, 1))?; + let local_feature_2 = local_feature_2.i((.., 1..))?; + Tensor::cat(&[local_feature_2, local_feature_1], D::Minus1)? + .contiguous()? + } + VisionModel::Qwen2(qwen2) => qwen2.forward(&local_feature_1)?, + }; + // let local_feature_2 = self + // .vision_model + // .forward(&image_crop_i, Some(&local_feature_1))?; + // let local_feature_1 = local_feature_1.flatten(2, 3)?.permute((0, 2, 1))?; + // let local_feature_2 = local_feature_2.i((.., 1..))?; + // let local_features = + // Tensor::cat(&[local_feature_2, local_feature_1], D::Minus1)? + // .contiguous()?; let local_features = self.projector.forward(&local_features)?; let global_features_1 = self.sam_model.forward(&image_ori_i)?; - let global_features_2 = self - .vision_model - .forward(&image_ori_i, Some(&global_features_1))?; - let global_features_1 = global_features_1.flatten(2, 3)?.permute((0, 2, 1))?; - let global_features_2 = global_features_2.i((.., 1..))?; - let global_features = - Tensor::cat(&[global_features_2, global_features_1], D::Minus1)?; + let global_features = match &self.vision_model { + VisionModel::Vit(vit) => { + let global_features_2 = + vit.forward(&image_ori_i, Some(&global_features_1))?; + let global_features_1 = + global_features_1.flatten(2, 3)?.permute((0, 2, 1))?; + let global_features_2 = global_features_2.i((.., 1..))?; + Tensor::cat(&[global_features_2, global_features_1], D::Minus1)? + } + VisionModel::Qwen2(qwen2) => qwen2.forward(&global_features_1)?, + }; + // let global_features_2 = self + // .vision_model + // .forward(&image_ori_i, Some(&global_features_1))?; + // let global_features_1 = global_features_1.flatten(2, 3)?.permute((0, 2, 1))?; + // let global_features_2 = global_features_2.i((.., 1..))?; + // let global_features = + // Tensor::cat(&[global_features_2, global_features_1], D::Minus1)?; let global_features = self.projector.forward(&global_features)?; let (_, hw, n_dim) = global_features.dims3()?; - let h = (hw as f32).sqrt() as usize; - let w = h; let (_, hw2, n_dim2) = local_features.dims3()?; - let h2 = (hw2 as f32).sqrt() as usize; - let w2 = h2; - let global_features = global_features.reshape((h, w, n_dim))?; - let image_newline = self.image_newline.unsqueeze(0)?.unsqueeze(0)?; - let global_cat = image_newline.expand((h, 1, n_dim))?; - let global_features = Tensor::cat(&[&global_features, &global_cat], 1)?; + let (global_features, local_features) = if let Some(image_newline) = + &self.image_newline + { + let h = (hw as f32).sqrt() as usize; + let w = h; + let h2 = (hw2 as f32).sqrt() as usize; + let w2 = h2; + let global_features = global_features.reshape((h, w, n_dim))?; + let image_newline = image_newline.unsqueeze(0)?.unsqueeze(0)?; + let global_cat = image_newline.expand((h, 1, n_dim))?; + let global_features = Tensor::cat(&[&global_features, &global_cat], 1)?; + let local_features = local_features + .reshape((height_crop_num, width_crop_num, h2, w2, n_dim2))? + .permute((0, 2, 1, 3, 4))? + .reshape((height_crop_num * h2, width_crop_num * w2, n_dim2))?; + let local_cat = image_newline.expand((height_crop_num * h2, 1, n_dim2))?; + let local_features = Tensor::cat(&[&local_features, &local_cat], 1)?; + (global_features, local_features) + } else { + (global_features, local_features) + }; + + // let h = (hw as f32).sqrt() as usize; + // let w = h; + // let (_, hw2, n_dim2) = local_features.dims3()?; + // let h2 = (hw2 as f32).sqrt() as usize; + // let w2 = h2; + // let global_features = global_features.reshape((h, w, n_dim))?; + // let image_newline = self.image_newline.unsqueeze(0)?.unsqueeze(0)?; + // let global_cat = image_newline.expand((h, 1, n_dim))?; + // let global_features = Tensor::cat(&[&global_features, &global_cat], 1)?; + // let global_features = global_features.reshape(((), n_dim))?; + // let local_features = local_features + // .reshape((height_crop_num, width_crop_num, h2, w2, n_dim2))? + // .permute((0, 2, 1, 3, 4))? + // .reshape((height_crop_num * h2, width_crop_num * w2, n_dim2))?; + // let local_cat = image_newline.expand((height_crop_num * h2, 1, n_dim2))?; + // let local_features = Tensor::cat(&[&local_features, &local_cat], 1)?; let global_features = global_features.reshape(((), n_dim))?; - let local_features = local_features - .reshape((height_crop_num, width_crop_num, h2, w2, n_dim2))? - .permute((0, 2, 1, 3, 4))? - .reshape((height_crop_num * h2, width_crop_num * w2, n_dim2))?; - let local_cat = image_newline.expand((height_crop_num * h2, 1, n_dim2))?; - let local_features = Tensor::cat(&[&local_features, &local_cat], 1)?; let local_features = local_features.reshape(((), n_dim2))?; Tensor::cat( &[ @@ -1256,21 +1401,44 @@ impl DeepseekOCRModel { )? } else { let global_features_1 = self.sam_model.forward(&image_ori_i)?; - let global_features_2 = self - .vision_model - .forward(&image_ori_i, Some(&global_features_1))?; - let global_features_1 = global_features_1.flatten(2, 3)?.permute((0, 2, 1))?; - let global_features_2 = global_features_2.i((.., 1..))?; - let global_features = - Tensor::cat(&[global_features_2, global_features_1], D::Minus1)?; + let global_features = match &self.vision_model { + VisionModel::Vit(vit) => { + let global_features_2 = + vit.forward(&image_ori_i, Some(&global_features_1))?; + let global_features_1 = + global_features_1.flatten(2, 3)?.permute((0, 2, 1))?; + let global_features_2 = global_features_2.i((.., 1..))?; + Tensor::cat(&[global_features_2, global_features_1], D::Minus1)? + } + VisionModel::Qwen2(qwen2) => qwen2.forward(&global_features_1)?, + }; + // let global_features_2 = self + // .vision_model + // .forward(&image_ori_i, Some(&global_features_1))?; + // let global_features_1 = global_features_1.flatten(2, 3)?.permute((0, 2, 1))?; + // let global_features_2 = global_features_2.i((.., 1..))?; + // let global_features = + // Tensor::cat(&[global_features_2, global_features_1], D::Minus1)?; let global_features = self.projector.forward(&global_features)?; let (_, hw, n_dim) = global_features.dims3()?; - let h = (hw as f32).sqrt() as usize; - let w = h; - let global_features = global_features.reshape((h, w, n_dim))?; - let image_newline = self.image_newline.unsqueeze(0)?.unsqueeze(0)?; - let global_cat = image_newline.expand((h, 1, n_dim))?; - let global_features = Tensor::cat(&[&global_features, &global_cat], 1)?; + let global_features = if let Some(image_newline) = &self.image_newline { + let h = (hw as f32).sqrt() as usize; + let w = h; + let global_features = global_features.reshape((h, w, n_dim))?; + let image_newline = image_newline.unsqueeze(0)?.unsqueeze(0)?; + let global_cat = image_newline.expand((h, 1, n_dim))?; + Tensor::cat(&[&global_features, &global_cat], 1)? + } else { + global_features + }; + // let (_, hw, n_dim) = global_features.dims3()?; + // let h = (hw as f32).sqrt() as usize; + // let w = h; + // let global_features = global_features.reshape((h, w, n_dim))?; + // let image_newline = self.image_newline.unsqueeze(0)?.unsqueeze(0)?; + // let global_cat = image_newline.expand((h, 1, n_dim))?; + // let global_features = Tensor::cat(&[&global_features, &global_cat], 1)?; + let global_features = global_features.reshape(((), n_dim))?; Tensor::cat(&[global_features, self.view_seperator.unsqueeze(0)?], 0)? }; diff --git a/src/models/deepseek_ocr/processor.rs b/src/models/deepseek_ocr/processor.rs index 5c2ae9b..5bf5d74 100644 --- a/src/models/deepseek_ocr/processor.rs +++ b/src/models/deepseek_ocr/processor.rs @@ -18,10 +18,11 @@ pub struct DeepseekOCRProcessor { image_token_id: u32, patch_size: u32, downsample_ratio: u32, + version: usize, } impl DeepseekOCRProcessor { - pub fn new(device: &Device, dtype: DType) -> Result { + pub fn new(device: &Device, dtype: DType, version: usize) -> Result { Ok(Self { device: device.clone(), dtype, @@ -29,6 +30,7 @@ impl DeepseekOCRProcessor { image_token_id: 128815, patch_size: 16, downsample_ratio: 4, + version, }) } @@ -70,6 +72,8 @@ impl DeepseekOCRProcessor { let mut images_seq_mask = vec![0u32]; let mut tokenized_id = vec![0u32]; let mut images_spatial_crop = Vec::new(); + let min_img_size = if self.version == 2 { 768 } else { 640 }; + let max_num = if self.version == 2 { 6 } else { 9 }; for (text_seq, image) in text_splits.iter().zip(imgs) { if !text_seq.is_empty() { let token_ids = tokenizer.text_encode_vec(text_seq.to_string(), false)?; @@ -79,10 +83,12 @@ impl DeepseekOCRProcessor { } if crop_mode { let mut images_crop_raw = Vec::new(); - let crop_ratio = if image.height() <= 640 && image.width() <= 640 { + let crop_ratio = if image.height() <= min_img_size && image.width() <= min_img_size + { (1u32, 1u32) } else { - let (img_crop, ratio) = dynamic_preprocess(&image, image_size, false)?; + let (img_crop, ratio) = + dynamic_preprocess(&image, 2, max_num, min_img_size, false)?; images_crop_raw = img_crop.clone(); ratio }; @@ -106,16 +112,25 @@ impl DeepseekOCRProcessor { let num_queries = image_size / self.patch_size / self.downsample_ratio; let num_queries_base = base_size / self.patch_size / self.downsample_ratio; - let mut token_repeat = num_queries_base.pow(2) + num_queries_base + 1; + let mut token_repeat = if self.version == 1 { + num_queries_base.pow(2) + num_queries_base + 1 + } else { + num_queries_base.pow(2) + 1 + }; if crop_ratio.0 > 1 || crop_ratio.1 > 1 { - token_repeat += (num_queries * crop_ratio.0 + 1) * (num_queries * crop_ratio.1); + let add_num = if self.version == 1 { + (num_queries * crop_ratio.0 + 1) * (num_queries * crop_ratio.1) + } else { + (num_queries * crop_ratio.0) * (num_queries * crop_ratio.1) + }; + token_repeat += add_num; } let tokenized_image = vec![self.image_token_id; token_repeat as usize]; tokenized_id.extend_from_slice(&tokenized_image); let seq_mask = vec![1u32; tokenized_image.len()]; images_seq_mask.extend_from_slice(&seq_mask); } else { - let global_view = if image_size <= 640 { + let global_view = if image_size <= min_img_size { image.resize_exact( image_size, image_size, @@ -130,7 +145,11 @@ impl DeepseekOCRProcessor { images_spatial_crop.push(vec![1, 1]); let num_queries = image_size / self.patch_size / self.downsample_ratio; - let token_repeat = num_queries.pow(2) + num_queries + 1; + let token_repeat = if self.version == 1 { + num_queries.pow(2) + num_queries + 1 + } else { + num_queries.pow(2) + 1 + }; let tokenized_image = vec![self.image_token_id; token_repeat as usize]; tokenized_id.extend_from_slice(&tokenized_image); let seq_mask = vec![1u32; tokenized_image.len()]; diff --git a/src/models/mod.rs b/src/models/mod.rs index 1c8230a..dc3fe87 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -10,6 +10,7 @@ pub mod hunyuan_ocr; pub mod mask_gct; pub mod minicpm4; pub mod paddleocr_vl; +pub mod qwen2; pub mod qwen2_5vl; pub mod qwen3; pub mod qwen3_5; @@ -70,6 +71,8 @@ pub enum WhichModel { Qwen3vl32B, #[value(name = "deepseek-ocr", hide = true)] DeepSeekOCR, + #[value(name = "deepseek-ocr2", hide = true)] + DeepSeekOCR2, #[value(name = "hunyuan-ocr", hide = true)] HunyuanOCR, #[value(name = "paddleocr-vl", hide = true)] @@ -110,6 +113,7 @@ impl WhichModel { WhichModel::Qwen3vl8B => "Qwen/Qwen3-VL-8B-Instruct", WhichModel::Qwen3vl32B => "Qwen/Qwen3-VL-32B-Instruct", WhichModel::DeepSeekOCR => "deepseek-ai/DeepSeek-OCR", + WhichModel::DeepSeekOCR2 => "deepseek-ai/DeepSeek-OCR-2", WhichModel::HunyuanOCR => "Tencent-Hunyuan/HunyuanOCR", WhichModel::PaddleOCRVL => "PaddlePaddle/PaddleOCR-VL", WhichModel::PaddleOCRVL1_5 => "PaddlePaddle/PaddleOCR-VL-1.5", @@ -140,6 +144,7 @@ impl WhichModel { | WhichModel::Qwen3_5Gguf => "vlm", // OCR models WhichModel::DeepSeekOCR + | WhichModel::DeepSeekOCR2 | WhichModel::HunyuanOCR | WhichModel::GlmOCR | WhichModel::PaddleOCRVL @@ -312,6 +317,10 @@ pub fn load_model<'a>( let model = DeepseekOCRGenerateModel::init(path, None, None)?; ModelInstance::DeepSeekOCR(model) } + WhichModel::DeepSeekOCR2 => { + let model = DeepseekOCRGenerateModel::init(path, None, None)?; + ModelInstance::DeepSeekOCR(model) + } WhichModel::HunyuanOCR => { let model = HunyuanOCRGenerateModel::init(path, None, None)?; ModelInstance::HunyuanOCR(model) diff --git a/src/models/qwen2/mod.rs b/src/models/qwen2/mod.rs new file mode 100644 index 0000000..82711f9 --- /dev/null +++ b/src/models/qwen2/mod.rs @@ -0,0 +1,275 @@ +use anyhow::Result; +use candle_core::Tensor; +use candle_nn::{ + Activation, Linear, Module, RmsNorm, VarBuilder, linear, linear_no_bias, rms_norm, +}; + +use crate::{ + models::common::{GateUpDownMLP, eager_attention_forward}, + position_embed::rope::{RoPE, apply_rotary_pos_emb}, +}; + +#[derive(Debug, Clone, PartialEq, serde::Deserialize)] +pub struct Qwen2Config { + pub vocab_size: usize, + pub hidden_size: usize, + pub intermediate_size: usize, + pub num_hidden_layers: usize, + pub num_attention_heads: usize, + pub num_key_value_heads: usize, + pub max_position_embeddings: usize, + pub sliding_window: usize, + pub max_window_layers: usize, + pub tie_word_embeddings: bool, + pub rope_theta: f32, + pub rms_norm_eps: f64, + pub use_sliding_window: bool, + pub hidden_act: Activation, +} + +#[derive(Debug, Clone)] +pub struct Qwen2Attention { + q_proj: Linear, + k_proj: Linear, + v_proj: Linear, + o_proj: Linear, + num_heads: usize, + num_kv_heads: usize, + num_kv_groups: usize, + head_dim: usize, + hidden_size: usize, + kv_cache: Option<(Tensor, Tensor)>, +} + +impl Qwen2Attention { + pub fn new(cfg: &Qwen2Config, vb: VarBuilder) -> Result { + let hidden_size = cfg.hidden_size; + let num_heads = cfg.num_attention_heads; + let num_kv_heads = cfg.num_key_value_heads; + let num_kv_groups = num_heads / num_kv_heads; + let head_dim = hidden_size / num_heads; + let q_proj = linear(hidden_size, num_heads * head_dim, vb.pp("q_proj"))?; + let k_proj = linear(hidden_size, num_kv_heads * head_dim, vb.pp("k_proj"))?; + let v_proj = linear(hidden_size, num_kv_heads * head_dim, vb.pp("v_proj"))?; + let o_proj = linear_no_bias(hidden_size, hidden_size, vb.pp("o_proj"))?; + Ok(Self { + q_proj, + k_proj, + v_proj, + o_proj, + num_heads, + num_kv_heads, + num_kv_groups, + head_dim, + hidden_size, + kv_cache: None, + }) + } + + pub fn forward( + &mut self, + xs: &Tensor, + cos: &Tensor, + sin: &Tensor, + attention_mask: Option<&Tensor>, + ) -> Result { + let (b_sz, q_len, _) = xs.dims3()?; + let query_states = self.q_proj.forward(xs)?; + let key_states = self.k_proj.forward(xs)?; + let value_states = self.v_proj.forward(xs)?; + let query_states = query_states + .reshape((b_sz, q_len, self.num_heads, self.head_dim))? + .transpose(1, 2)?; + let key_states = key_states + .reshape((b_sz, q_len, self.num_kv_heads, self.head_dim))? + .transpose(1, 2)?; + let value_states = value_states + .reshape((b_sz, q_len, self.num_kv_heads, self.head_dim))? + .transpose(1, 2)?; + let (query_states, key_states) = + apply_rotary_pos_emb(&query_states, &key_states, cos, sin, false)?; + let (key_states, value_states) = match &self.kv_cache { + None => (key_states, value_states), + Some((prev_k, prev_v)) => { + let key_states = Tensor::cat(&[prev_k, &key_states], 2)?; + let value_states = Tensor::cat(&[prev_v, &value_states], 2)?; + (key_states, value_states) + } + }; + + self.kv_cache = Some((key_states.clone(), value_states.clone())); + let scale = 1f64 / f64::sqrt(self.head_dim as f64); + let attn_output = eager_attention_forward( + &query_states, + &key_states, + &value_states, + Some(self.num_kv_groups), + attention_mask, + scale, + )?; + let attn_output = attn_output.reshape((b_sz, q_len, self.hidden_size))?; + let attn_output = attn_output.apply(&self.o_proj)?; + Ok(attn_output) + } + + pub fn forward_no_cache( + &self, + xs: &Tensor, + cos: &Tensor, + sin: &Tensor, + attention_mask: Option<&Tensor>, + ) -> Result { + let (b_sz, q_len, _) = xs.dims3()?; + let query_states = self.q_proj.forward(xs)?; + let key_states = self.k_proj.forward(xs)?; + let value_states = self.v_proj.forward(xs)?; + let query_states = query_states + .reshape((b_sz, q_len, self.num_heads, self.head_dim))? + .transpose(1, 2)?; + let key_states = key_states + .reshape((b_sz, q_len, self.num_kv_heads, self.head_dim))? + .transpose(1, 2)?; + let value_states = value_states + .reshape((b_sz, q_len, self.num_kv_heads, self.head_dim))? + .transpose(1, 2)?; + let (query_states, key_states) = + apply_rotary_pos_emb(&query_states, &key_states, cos, sin, false)?; + + let scale = 1f64 / f64::sqrt(self.head_dim as f64); + let attn_output = eager_attention_forward( + &query_states, + &key_states, + &value_states, + Some(self.num_kv_groups), + attention_mask, + scale, + )?; + let attn_output = attn_output.reshape((b_sz, q_len, self.hidden_size))?; + let attn_output = attn_output.apply(&self.o_proj)?; + Ok(attn_output) + } + + pub fn clear_kv_cache(&mut self) { + self.kv_cache = None + } +} + +#[derive(Debug, Clone)] +pub struct Qwen2DecoderLayer { + self_attn: Qwen2Attention, + mlp: GateUpDownMLP, + input_layernorm: RmsNorm, + post_attention_layernorm: RmsNorm, +} + +impl Qwen2DecoderLayer { + pub fn new(cfg: &Qwen2Config, vb: VarBuilder) -> Result { + let self_attn = Qwen2Attention::new(cfg, vb.pp("self_attn"))?; + let mlp = GateUpDownMLP::new( + vb.pp("mlp"), + cfg.hidden_size, + cfg.intermediate_size, + cfg.hidden_act, + false, + None, + None, + None, + )?; + let input_layernorm = + rms_norm(cfg.hidden_size, cfg.rms_norm_eps, vb.pp("input_layernorm"))?; + let post_attention_layernorm = rms_norm( + cfg.hidden_size, + cfg.rms_norm_eps, + vb.pp("post_attention_layernorm"), + )?; + Ok(Self { + self_attn, + mlp, + input_layernorm, + post_attention_layernorm, + }) + } + + pub fn forward( + &mut self, + xs: &Tensor, + cos: &Tensor, + sin: &Tensor, + attention_mask: Option<&Tensor>, + ) -> Result { + let residual = xs; + let xs = self.input_layernorm.forward(xs)?; + let xs = self.self_attn.forward(&xs, cos, sin, attention_mask)?; + let xs = (xs + residual)?; + let residual = &xs; + let xs = xs.apply(&self.post_attention_layernorm)?.apply(&self.mlp)?; + let xs = (residual + xs)?; + Ok(xs) + } + + pub fn forward_no_cache( + &self, + xs: &Tensor, + cos: &Tensor, + sin: &Tensor, + attention_mask: Option<&Tensor>, + ) -> Result { + let residual = xs; + let xs = self.input_layernorm.forward(xs)?; + let xs = self + .self_attn + .forward_no_cache(&xs, cos, sin, attention_mask)?; + let xs = (xs + residual)?; + let residual = &xs; + let xs = xs.apply(&self.post_attention_layernorm)?.apply(&self.mlp)?; + let xs = (residual + xs)?; + Ok(xs) + } + + pub fn clear_kv_cache(&mut self) { + self.self_attn.clear_kv_cache() + } +} + +pub struct Qwen2Decoder { + layers: Vec, + norm: RmsNorm, + rotary_emb: RoPE, +} + +impl Qwen2Decoder { + pub fn new(vb: VarBuilder, cfg: &Qwen2Config) -> Result { + let mut layers = Vec::with_capacity(cfg.num_hidden_layers); + let vb_l = vb.pp("layers"); + for layer_idx in 0..cfg.num_hidden_layers { + let layer = Qwen2DecoderLayer::new(cfg, vb_l.pp(layer_idx))?; + layers.push(layer) + } + let norm = rms_norm(cfg.hidden_size, cfg.rms_norm_eps, vb.pp("norm"))?; + let head_dim = cfg.hidden_size / cfg.num_attention_heads; + let rotary_emb = RoPE::new(head_dim, cfg.rope_theta, vb.device())?; + Ok(Self { + layers, + norm, + rotary_emb, + }) + } + + pub fn forward_no_cache( + &self, + xs: &Tensor, + attention_mask: Option<&Tensor>, + seqlen_offset: usize, + ) -> Result { + let seq_len = xs.dim(1)?; + let (cos, sin) = self + .rotary_emb + .forward(seqlen_offset, seq_len, xs.device())?; + let mut xs = xs.clone(); + for layer in self.layers.iter() { + xs = layer.forward_no_cache(&xs, &cos, &sin, attention_mask)?; + } + let xs = xs.apply(&self.norm)?; + Ok(xs) + } +} diff --git a/src/position_embed/rope.rs b/src/position_embed/rope.rs index e4bb887..0f18e01 100644 --- a/src/position_embed/rope.rs +++ b/src/position_embed/rope.rs @@ -566,11 +566,13 @@ impl RoPE { let positions = Tensor::arange( seqlen_offset as f32, (seqlen_offset + seq_len) as f32, - device, + self.inv_freq.device(), )? .reshape((seq_len, 1))?; // (seq_len, 1) let freqs = positions.matmul(&self.inv_freq)?; // (seq_len, dim / 2) - let emb = Tensor::cat(&[&freqs, &freqs], D::Minus1)?.contiguous()?; // (seq_len, dim) + let emb = Tensor::cat(&[&freqs, &freqs], D::Minus1)? + .contiguous()? + .to_device(device)?; // (seq_len, dim) let cos = emb.cos()?; let sin = emb.sin()?; Ok((cos, sin)) diff --git a/src/utils/img_utils.rs b/src/utils/img_utils.rs index 0acb084..bd0d5c0 100644 --- a/src/utils/img_utils.rs +++ b/src/utils/img_utils.rs @@ -162,6 +162,8 @@ pub fn find_closest_aspect_ratio( pub fn dynamic_preprocess( image: &DynamicImage, + min_num: u32, + max_num: u32, image_size: u32, use_thumbnail: bool, ) -> Result<(Vec, (u32, u32))> { @@ -169,7 +171,7 @@ pub fn dynamic_preprocess( let orig_height = image.height(); let aspect_ratio = orig_width as f64 / orig_height as f64; // 控制分块数量在2-9之间 - let target_ratios = generate_target_ratios_sorted(2, 9); + let target_ratios = generate_target_ratios_sorted(min_num, max_num); let target_aspect_ratio = find_closest_aspect_ratio( aspect_ratio, &target_ratios, diff --git a/tests/messy_test.rs b/tests/messy_test.rs index 83608e7..43ab670 100644 --- a/tests/messy_test.rs +++ b/tests/messy_test.rs @@ -9,7 +9,7 @@ // use aha_openai_dive::v1::resources::chat::ChatCompletionParameters; use anyhow::Result; // use byteorder::{LittleEndian, ReadBytesExt}; -use candle_core::Tensor; +// use candle_core::Tensor; use modelscope::{DownloadOptions, ModelScope}; // use sentencepiece::SentencePieceProcessor; // use zip::ZipArchive; @@ -35,12 +35,19 @@ async fn download_test() -> Result<()> { #[test] fn messy_test() -> Result<()> { // RUST_BACKTRACE=1 cargo test -F cuda --test messy_test messy_test -r -- --nocapture - - let device = &candle_core::Device::Cpu; - let t1 = Tensor::randn(0.0, 1.0, (16, 9, 64, 128), device)?; - let t2 = Tensor::randn(0.0, 1.0, (16, 9, 128, 64), device)?; - let out = t1.matmul(&t2)?; - println!("out shape: {:?}", out); + let save_dir = + aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; + let model_path = format!("{}/deepseek-ai/DeepSeek-OCR-2/", save_dir); + let stem = std::path::Path::new(&model_path) + .file_stem() // 获取文件名主干（不含扩展名） + .and_then(|s| s.to_str()) + .unwrap_or("qwen3.5"); + println!("stem: {:?}", stem); + // let device = &candle_core::Device::Cpu; + // let t1 = Tensor::randn(0.0, 1.0, (16, 9, 64, 128), device)?; + // let t2 = Tensor::randn(0.0, 1.0, (16, 9, 128, 64), device)?; + // let out = t1.matmul(&t2)?; + // println!("out shape: {:?}", out); // let input = Tensor::arange(0.0f32, 25.0f32, device)?.reshape((5, 5))?; // println!("input: {}", input); diff --git a/tests/test_deepseek_ocr.rs b/tests/test_deepseek_ocr.rs index 5aefd5e..8129eec 100644 --- a/tests/test_deepseek_ocr.rs +++ b/tests/test_deepseek_ocr.rs @@ -5,6 +5,55 @@ use aha_openai_dive::v1::resources::chat::ChatCompletionParameters; use anyhow::Result; use rocket::futures::StreamExt; +#[test] +fn deepseek_ocr2_generate() -> Result<()> { + // RUST_BACKTRACE=1 cargo test -F cuda --test test_deepseek_ocr deepseek_ocr2_generate -r -- --nocapture + let message = r#" + { + "model": "deepseek-ocr2", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image", + "image_url": + { + "url": "file://./assets/img/ocr_test1.png" + } + }, + { + "type": "text", + "text": "\nConvert the document to markdown. " + } + ] + } + ], + "metadata": {"crop_mode": "false"} + } + "#; + let save_dir = + aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; + let model_path = format!("{}/deepseek-ai/DeepSeek-OCR-2/", save_dir); + let mes: ChatCompletionParameters = serde_json::from_str(message)?; + let i_start = Instant::now(); + let mut model = DeepseekOCRGenerateModel::init(&model_path, None, None)?; + let i_duration = i_start.elapsed(); + println!("Time elapsed in load model is: {:?}", i_duration); + let i_start = Instant::now(); + let res = model.generate(mes)?; + let i_duration = i_start.elapsed(); + println!("generate: \n {:?}", res); + if let Some(usage) = &res.usage { + let num_token = usage.total_tokens; + let duration_secs = i_duration.as_secs_f64(); + let tps = num_token as f64 / duration_secs; + println!("Tokens per second (TPS): {:.2}", tps); + } + println!("Time elapsed in generate is: {:?}", i_duration); + Ok(()) +} + #[test] fn deepseek_ocr_generate() -> Result<()> { // RUST_BACKTRACE=1 cargo test -F cuda --test test_deepseek_ocr deepseek_ocr_generate -r -- --nocapture diff --git a/tests/weight_test.rs b/tests/weight_test.rs index 36f3c17..643b01d 100644 --- a/tests/weight_test.rs +++ b/tests/weight_test.rs @@ -243,3 +243,25 @@ fn index_tts2_weight() -> Result<()> { // } Ok(()) } + +#[test] +fn deepseekocrv2_weight() -> Result<()> { + // cargo test -F cuda --test weight_test deepseekocrv2_weight -r -- --nocapture + let save_dir = + aha::utils::get_default_save_dir().ok_or(anyhow::anyhow!("Failed to get save dir"))?; + let model_path = format!("{}/deepseek-ai/DeepSeek-OCR-2/", save_dir); + let model_list = find_type_files(&model_path, "safetensors")?; + + let device = Device::Cpu; + for m in &model_list { + let weights = safetensors::load(m, &device)?; + for (key, tensor) in weights.iter() { + if key.contains("qwen2_model") { + println!("=== {} === {:?}", key, tensor.shape()); + } + // println!("=== {} === {:?}", key, tensor.shape()); + } + } + println!("model_list: {:?}", model_list); + Ok(()) +}