From 3c6cf1d400041508d7dbcce64e07f9d1334bc877 Mon Sep 17 00:00:00 2001
From: Justin John <34035011+justinjohn0306@users.noreply.github.com>
Date: Fri, 13 Oct 2023 08:26:49 +0530
Subject: [PATCH] Add files via upload

---
 .../__pycache__/generate_batch.cpython-38.pyc | Bin 0 -> 3495 bytes
 .../generate_facerender_batch.cpython-38.pyc  | Bin 0 -> 4073 bytes
 .../test_audio2coeff.cpython-38.pyc           | Bin 0 -> 3921 bytes
 .../__pycache__/audio2exp.cpython-38.pyc      | Bin 0 -> 1294 bytes
 .../__pycache__/networks.cpython-38.pyc       | Bin 0 -> 2149 bytes
 .../src/audio2exp_models/audio2exp.py         |  41 +++++
 .../src/audio2exp_models/networks.py          |  74 +++++++++
 .../__pycache__/audio2pose.cpython-38.pyc     | Bin 0 -> 2866 bytes
 .../__pycache__/audio_encoder.cpython-38.pyc  | Bin 0 -> 2176 bytes
 .../__pycache__/cvae.cpython-38.pyc           | Bin 0 -> 4699 bytes
 .../__pycache__/discriminator.cpython-38.pyc  | Bin 0 -> 2464 bytes
 .../__pycache__/networks.cpython-38.pyc       | Bin 0 -> 4751 bytes
 .../__pycache__/res_unet.cpython-38.pyc       | Bin 0 -> 1924 bytes
 .../src/audio2pose_models/audio2pose.py       |  94 +++++++++++
 .../src/audio2pose_models/audio_encoder.py    |  64 ++++++++
 .../SadTalker/src/audio2pose_models/cvae.py   | 149 ++++++++++++++++++
 .../src/audio2pose_models/discriminator.py    |  76 +++++++++
 .../src/audio2pose_models/networks.py         | 140 ++++++++++++++++
 .../src/audio2pose_models/res_unet.py         |  65 ++++++++
 19 files changed, 703 insertions(+)
 create mode 100644 animation/SadTalker/src/__pycache__/generate_batch.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/__pycache__/generate_facerender_batch.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/__pycache__/test_audio2coeff.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2exp_models/__pycache__/audio2exp.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2exp_models/__pycache__/networks.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2exp_models/audio2exp.py
 create mode 100644 animation/SadTalker/src/audio2exp_models/networks.py
 create mode 100644 animation/SadTalker/src/audio2pose_models/__pycache__/audio2pose.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2pose_models/__pycache__/audio_encoder.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2pose_models/__pycache__/cvae.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2pose_models/__pycache__/discriminator.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2pose_models/__pycache__/networks.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2pose_models/__pycache__/res_unet.cpython-38.pyc
 create mode 100644 animation/SadTalker/src/audio2pose_models/audio2pose.py
 create mode 100644 animation/SadTalker/src/audio2pose_models/audio_encoder.py
 create mode 100644 animation/SadTalker/src/audio2pose_models/cvae.py
 create mode 100644 animation/SadTalker/src/audio2pose_models/discriminator.py
 create mode 100644 animation/SadTalker/src/audio2pose_models/networks.py
 create mode 100644 animation/SadTalker/src/audio2pose_models/res_unet.py

diff --git a/animation/SadTalker/src/__pycache__/generate_batch.cpython-38.pyc b/animation/SadTalker/src/__pycache__/generate_batch.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5a80ca8fe5b37543fd7866d2c7262b6d9e6d22af
GIT binary patch
literal 3495
zcmZuz-D@1l5%2Dq-JM<SuD-R>N|t@(?ue3$5^w^EF}C3x5kfE_r+eAx7{=4HtC99g
zJuO)>JrG!%vmXwUR};eO@!mrIg8Tu=KhRHsgF6Iy$a4rJzv^9CQhYshPj&TY{i>>~
zKCRU%45jz%gCv<}>?`VAFFrbtP_t7sl1biWE&7Wt@9`Gr6P{=ZH|Di`_b;>x?qBSc
zTETQqxmOX4v))W=1{_cNFIcN83$loRO_pSUe_fVk1^-z&D{Hd;g16=*|AcMLUEp-~
zWQ$v0AIV-Us5>o07g0Y(&3=JKGs8yQ@IzlU4en?7OR>gs*8kqy5vCq%ejqT4c@yp`
z>j)_%m);kCjva8S|3q=@XM=dq&vew+Tf+LiflTb|B_}+Iy2nYj<yo(r^sVR*tv8G$
zCVJ1J6I+drWjY8k)Y1E2GBtzk{{5Se@9lk>B`VuHIL>ss_jvHE-yK9U+xvLX8>%GB
z61n$DBzL3kVWNJ}i_-p8cP~@%UMJ}%D$+^#G}7_@_VCQsV>K9tI5~7Dl1?+HSYG9I
z)J?8x;Kd%A%TfSAeFxlkK^ShtI?Fu*#CYofG510H##<tqxPDsz7*3k-QUFp=bcz+3
zvR-?bwdzk(9S#!}wpG+iGF#~%_olQHe$rHF&B8EJSu(lr6d@QO%K|iv2cpWWLM?!f
zNeP#Q?6HjMF8v&YW@AozY>RWAb18CRgob8E{IIA_q?da}7_Y&k-+%y;ud+i9KJ1Qc
z4*Cx8FUmrvOybkbtNAZs5p8k*8YdS=3r8nM3x=$+{N#eJ=8WXx*Pt%wX5XNq{NrKg
zt}9<X?6h6^`r$W!{~MKmKHTzD33F`Wd7=iHEvTs9Nxt+I0YE*S4qAbOFifQ_!1$DY
z2fr&T=%)R{FiVcsuskEk7?kcq?LJ@P+q^D(u5N+*4w8hmL-J1`ZXyY+1dwWyq(JdH
zs&BM6_O;Lj=>uo{brG`*WH7Qg5+l;YCF-SXsJ{W$@Nd+hQ}{o$F%M{a*U$zBXm1|9
zIofn+&pc6!xaGwXnhE4&dKcvW7dzCDJ*yRJ8fZGGDF1r6RdQIXRrC}gWQ%}X4tlm2
z?+?;Av0g9jt2zxgsUau5W~b}#;v7yk=$?ImsTmzffCqq^Kz|>ge>egCLyYX~#?GW^
z7Rnt|(mi=z_GjE)@;rdqmNh%zW8qrQuyZ^_Yt;I9x+*uu_>Hj{Q&3N}Z;DblFRkjr
zxM=8M<!uj&sP+G<OXI*mHRmF)>$0wxIw?wuk+&!E*}SS}Ox?_mXtXr0n)*3EsJ&*d
z*tmYq^0^MnYkF4CnJQM5M&dF1Oy=`uUIwcyU(miRn*}{@$|Jtc+zl4Zq6wh9CB1-E
zi)INX!Sm(MS(`)oGcN+O{FvU8Wiw}%5hkGcHa*ZQrZ!$QD+jf6gq5SeK#nB?8mBMQ
z*?GgTd{wWE8)g;UHM1tGYZQKJFdQc9(_NanaIk8cK){!?UwC=bc#<8Ke=pQWFxLjc
zLG>kE+U-O5tef@eK4gGQ&Q11t&0q0@HHZF&*)Y{Zh8+S^H5*Nq-=hBt%QrvgkZGP|
zf~2=b+{ty@+#WB!;y7vZ`i>+O?$$K7-Oa>nuGcTm7-7t<z}>FyV49$>-3qtf7~djH
z#+!1|`h<@u;chKA)^KCQZS+@ea>lpi0`VQU`F36tsNXT<vNh9~QVr9bQWzyHO~cHX
zid_6-5zpSzlZ!bR?*;kN#eY$pH!AYW^mh*!d+zs=?mcw}{ize~KDzPpTt;ki=ZsvW
zgO~ZU%^Z<fBOXh<DX-W8fPi}OUXR2d#F3AK#pEDbh6j{<53U=X;(gqYh6!S>*#43A
zQ=Rk>o`Qi&J0Wgz&6O+nyJ@ClMA-+wqO&QO6q#VpH#&(9Wu&9+``tktb+ZQ`E~nZ`
zJV@H@@UHCj+!LhUqZ=R>iB8k(;t@LUh)3S-#3Q7fctOrzv56p8FAPNb2oo$%vE95%
z+R0h+)V=2J$g8WXA7WaF`u5IKXmOFEt)zXKo{+Ga^{~q%jsz}_gdJ@wlQg7{kZglt
z8izz}y=Mhtk3Wp`z74Wr7r>orfJxNkIiz>Wy_>e+c4!5Zr-aCCTwqp|cDcNwRj0Fu
zNvzeNgUE0BuJz-=@Jx}eC$Dayjm6sfgsUw^!(q~wwiIQNQqh?rGq8RZrvqD}LwcB6
zRLL-jwE8}c%^>B9k@NIXZDFX8eb|B?sCeJb?9k4;Nk1DXTR!e*N5@I>9JAVKw;Nia
z2U~aSTsu{n4kxN~GEY%)bWtWJ6bc$ZFTAM)WK{qNF&LY*f>d@BN&#E5lcX*jv?nP{
zt9*=X#woA0oSpUKK1>Rzgqco;VU{A&lP##9QS(!3DA=o?P(x`$s{&Z3c@TgAmg+9C
zXI=20<WM29DO;W}Qh91ifDj47sr3;2E$^vyTCtoBj#Zo>A?P7zP2<c-@GWea#=2D|
zi5)7fm3Is<bW?8duM#@z!RV||$+s;eEj#mtiEopPKc>bde!-=)k$hPn4MT$Biw(Zv
zQT$v$>bJxzpb$~T0@@|iI$z*(Xe)x^Y=y7#IYD!Xs^AOot?(vh)W!Q^MHI2mil908
zgy7SUiT|kI;>0a~XJ==3YhK+$XJVzl1Z4}54`%Bu|2xD8V3WhMZP=wq2kPfwK_x2Q
zK88JH+s-X)!F3egzKJ_byn`|>XV>n)E2qEmY8%{#Egg502b4#;M1t~O5+!gz1f?Jd
HszLpKjr~}k

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/__pycache__/generate_facerender_batch.cpython-38.pyc b/animation/SadTalker/src/__pycache__/generate_facerender_batch.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6f4e38c308dd40f08e904beddcc767040b6410f9
GIT binary patch
literal 4073
zcmai0&2Jn@6|d@#nVwINzsHWf@p_Xb1KP0J1QvqjT`5r@5?I+y!Y(6+rZd&!v^`(0
zE@LOEzTgCw!;0*U<M_B2g!l{iJNk+Qvc!S&9u|JDdd7AVOU$TWRlRzz>ec&xSg8~l
ze%<}cVKT?qKWK6Gm&f8`eDOAbV1jp8GaGr+!rJQCU8m`A!r2|Sn``E}`DVWBH9dpl
zokFu<_=?yo0ap{AC_LxQvM7qubJnbgvZ!FJimI5vI3f6b)|foTeQcw_Q}=E+*a{nV
zYDN9D8g;k)AokmxexR;@kQS5-dU3liyZ<73#{Tosoz!W`{@|E###-eJYv8+p1$?*g
z#cdnU;YnGsZ9cRNv^6{8!df6`FR-MN?HMess}V~klrwZSKjKME<%W4(!)#J{dh(FQ
z!my|(bxD_m)3%SSWKvHKOTslY<zYon9`RvS*jP{Lso{inM2`MP4z2Zs$geR~8%}Bq
z6w_t~S&|vy>Edwei0$O%KlQXIEReEuFG&5x7gie*C$oBHglDhd=X&qI#!>oz9BNw6
z>hhtL%qx<4{zT@wu8(CtvJ<Qzae0mDImnUSTFK7V-pm`=HFolf+)FP1s&d;idJb?x
z&yM(ubPByj&u!0=1%xfCU)oz1bX5J)O6GI{od14>r*b~eiA<w?=+)J5uX(uNq4lRb
z$%3A{%^uxM7WE>=OUVVjFq~8K`obZHG?OnNg_r{M^gJxU*-LBJBMLDyTCvaA57;N{
zQ}#LAV#(52;$&GbLo2hmudbI5InG{6-cq<tzXhGz$wj?1T+|mwrwC9xb1YfWE5hEf
zUt02S;B)j!of+O*^7bqAL*sKx@QGEuI=le?vGgi*H3yy1NWSYNtHy4tM;wyO!#)*^
zb&NK~1&pp>JB63FTvSWiB?%zcJBH^S!?OsQ+=yl!G`SJYB|~$`&|Cmbenj&aH2D$D
zWkYk>&@6#w0-?OTaa=*j9sd_UiJ)Z1h&|S@k1Y%tg0PZ^(sJD2ldaH?DCkqCz1Qia
z#a2ITxBcs)+daMmqT{<nSsPKtVK?Zhs1+Z73^Wpk6C6bGGv}&0P$}O(rVRGXT3Pvq
z7gAnK3vC&6L%+AzJ^mH~rL!uKTcPq#j`2HDtkU9dBN6uf9%$2IuqUFvKM2&5wD2_8
z_gkReO-qAFwVsR?McMCkvUy`Uwfb@D7<w-rbRrc(6xYloJvw#zgRqxcy+P^*QU*_@
z1FUpuD<{J^dKQi{+n^tZsjK?3^&~Cbp``a9?8SYV7WR7a?p_#z5*&jtP^lBQkRkIR
z5?w6aPJcg?>C8!nJb9vjEiJYBy;h*Y9zIC+Bp6`Wmyk5IV<l4;X%+fkIEOjYSDFh3
zc!Nmukk_zJY=u3)1%1iDC$Ssvjt!S7Vi2`571r93jFq1mmG<OY^zh>2fYfOoUyXW$
zJ>`?GjWuf7I#am8Ok^0N^^7Du7(kRb>i7J(6?9;>SVgc&w=Xcq-az(SAb_4ozY}AF
z`!W#GUM!a(MAL)QWKudJP(kXT-dvMa+L}1)nZAnC+1DM=7g0+!tIz{<ph98Ra+;XW
zAjyrJGjH(r!u{s-8(3V!+o@}GmGVd~;V`)fF!tj}9Hh0^T{taB1Vz*nus`>k)35LQ
zy@9+)C#?{88{lguKgICPqgyvOAI71KH@EkozRlbH{T?|~ym_nNHTn(3=KVlC2s%5V
ze2*?UU2ev*wK;yh-ws-#golLEH<rCRc$!XAq@CFy9ecPpRshU#_!6&Kj#cDE?(r$W
z8vY)wtun9kMZ3=HSk-uyd-fHp%5PXX{zD5i_^)EmF`Q*y=2gox{8i8|5xqGFcwkSU
zB_H747RBWFN0CT@NI5G05x^f<n+x;>&pm~Tq`7crm~iLWp>-QsI<XY`2W_FMTZygh
zp)FjRBk$!lT9ntuKe~htC4Y!(09#D6M_$6eU|(6g&i%{+#taQBwXVJ|e*vPedFqWl
z)$Gapz^27Da{3l*vi>VNkEG5qi#@w?PQ+fjy4L9<B;s{U#%w0>O6reH;%zwcLz;hw
zz;y!O1pxLln*S7_SsBY=Vp7r*GKqdkGZIUF0Ps1!I0wM^3N&qSEqeI)M|&q4Ca=?M
z8@<>9`y*3aS(<CB&IqFZ;ybjUW!qT67S3}|+eYt>W~hIr^DrHdt6k{c65QzC-N@cY
zHRTN}kO0OCE0`E7)Dy07IgE?zzJM?&z*c6JhIM)p>|%GL9zbva{6nxKL#VI|D-}^u
zd{bNEQ#(mg#&7VQ8tA=q43x#^c~*h}ZkSuhZkdIQU2;bKX(=ma6zK=XNXXu(Os+vR
z%cphI-^zsA#9^7K$h9AlWTyW|P%I+gYFrZj6?2gaDMoJ*@n7K4Xn1S|qoGyeJC>BB
zH`A_6B@P0>(l8q}(VXqp$&6@lU>srX*m4uJmUc(&&6wV%X3x<7OxZ)6ftJ%9g;Y-G
z?la*ed6mPtun)CnS!1<nC0<g{d6m}%ID@yq=-dxklHdJ38WKoXXBSvfe1mg+#A$ev
zk}jGPO2m!EwV*KrUs>DAgFILRS3a^2`8-~}`**w@bIo@gw4+Pf7I0RQDzSSvPWBk~
z?vZ9P%;?24BMqIr_$A-HsfroRAvC6nq)Tqx|5R@{>(8%f?!0q;cfRf#2n8}>=EY>}
zEqvzJD5Z8M>}78CW1zC=ze2N{0BIqFQ=3|r+QH*k{+f`vAnOg>fsA^pQE5(%ioOW9
zWEf`2fl8G82%tHE!segd`h>Rb(N^WmR&UTOj;P|ju1S4qE<-eHqm2PdiE;R5X?$Q1
ztCK~nP~s_{oQc1mU@7p$>i`Vpfl5PGE~vCjaR<0MpEKpf!>o=1LH`--l~FRXA~J=Q
zq_Z0Njg1XS3FMBX#!^ybG-`4cL+V0XgQrG`seSM6&$C`HAMcoER#FZ;ZS>^#h{Z$C
zHh6k9>dWubE>$CwFijYm8_&GRyv?M4Zw+0|UMF0q0u!$QPysAMYQQYlj!VzC#e1IX
G<^Kneoe(|%

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/__pycache__/test_audio2coeff.cpython-38.pyc b/animation/SadTalker/src/__pycache__/test_audio2coeff.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2c2726877d509a40e17371f5f8f8b98c650f6fa6
GIT binary patch
literal 3921
zcmaJ^TW{RP6`mPhmdm~9E=zV3I7ppl(?%)M7ARW7up=jKi$F+gr%nk41WR&Pl(^)2
zhO#Yz`w}^*Q^2VE-iKYZ5A|Q-Yo7%47xX3l&QS8MYNr%9J9Fmn%$b?<edp-&PRC_<
zhL5hrOKXh%iw5(Lfx%6@>ThTw7V(sM^)L6h9t+RFuSm_I<yk}9vpLNhX=CVkP2D$B
zci8e;y3f<L*Vf;4^skk6hYQ|<p0m@%;gYwc`;ByYxZ<tozLTyF*St0KyOA5Uo^Wsd
z4vX4R=Lw5CC&IfF@w=?Ia0(CEPES+~6+DcGK~Xi*Jc!Wg@v5=iAME5&jBW?hgFN;7
zNm|4b0{7Nol;j_NmMfZGxSI?|X?$M>qfsn>{~<)pTH)izBMjEbsQ=jS%Y5jopdS};
zrgB*s;pi|l$%nJY!Y{qsc-61a6l|ZDXeXjHraa=$#MmrxhzCviUdbm$X+~m`O*v<!
z(9)Zw&{E?}YDVTJ!&a>w_|YB)BR|c<ph)tpvWD1PS~c@gkqndLSXNd%%EP^$Sy@Fc
z!@bHRzbiY6A0}a3E$CgSA}C@%O2VRNc?;oQ93G7FBrE(;Q0&Pze3l(*me5r0Z2kq4
zs}TL?-u5qcKUc9-yZeW#NOo`Mk1{f>cDM84NXAOV(eB+Kx)-Dev3!3RB-w1Zt7N!a
z#H#QEeg0t{_xoF;qsq~~g`<N-tf9yY##gz+yYvdVj&VrB`R8Svz5{8@C^kJ2@kl&j
zh)*XnBMWh3Mt0PI#EJ|YW3y`27rRX>hh*aXAyC+O_1*o0)7{~w_$E8yW24~p4dV99
zi7{oop2gi^*3z@ql#lJwKAVMJM0aXDGfL)LCEK?qb}6QahO<$FtZ2D2Ww+Uv?zj<|
zWy5bi$c}GgWowhke-`e3YtqKZrg#`TTEA0tODAfaz#3=SYO#QMXH(n4ip^&RXZIoJ
zun#{MBlm=V#s9!E@dYcEPT06vHfvvWZ+`VAR{u9<wQt8C)m}X3a_bd8nv>d#)|nT}
z+KV>4Xuj5m&YTa+(7{>Yg1XO4oW}}wwes?G>QU`g`}|Cv^9NsPt@_jizFLh47YjSy
zY8^`>VA($khN+HZZ>5&Tk4H0MkL$4hr0O0*cLrp+f_wMenH_I!*7s)^=!qI2gpK1L
zd=eyS99>Pas}#IRHn{rQkG$qfPtIK>r)E}<-}x>}7xl0@E$poUWurjGejx!atP4s|
zs<&2|`6$jRvAyG+!Kpu}?7oa+a05klwK(TsO&+4iD>Hr=q?HSk!4S+QRn?L4gTn+^
z_6HIK(wd!=k_%+0so@GXjpZ`NvPlhrv}%6*IE+C^c_teqbV#`H@&tUntlme@txOfC
z{i>^isL*UtbCBL-vmf>cKHaJ3%z9dgQ}G%kL#MK|+f{3JoXRt^xVWR)s&0_vSmpRW
z4y*9}D;Q~RW}?kq@QZGZMR1Kvd?N&8i+ojVaGQ4x8`3r;4j1Sx@@273a=wh0zHr6y
z>Mc!Xno_pTm)Qv^5u887XhblYTp~>_6Fz0n`CmTMBtWFk0dT9dh;qg@QifR?1IQ*s
z9yW*r%yykX-Z;~3l#K{!0;J&VgYD1*up{e$$w@&ybAv_p6%bTpud>Lzgn3wXi&oK|
zbV_H^oq|%JX9I|hXXb3Bv6H>~s%B)pP7?`k0bq9e0*u>4dUH!RayQvzampr3c(KZ4
zdCJBuBE*9x$OI%dSt&7|igBx0E!OteCzn76E$om8gh!oC))(t+Dxz-LO8AKYjr||3
z`%JK(vEQ=avA=^7==TfMBQ-g6HHL`neZVHFtS}9fs+yWDNOFqHV5gMwP6e75yfTBw
zi8}oj4^plXB|}Ln$L=-t_S@Ig7F8L}C<*-xa5*Cl-AA_6>-;1d4tpClP#RgTF(kha
z6O~D+uW~=lv%x)GLC7~C)@YF5BLz$C1*5n!kdG_-OohUz5*b20kTN)u6h^2#;05-Q
zXEEjXB0#pcvcn-#Y3EQ8C&;)@dz7@PO600CDZy9fK9YT}EwAb2{umVDp4YAYrV<K;
zfdmy0m8#+>jjI*d(J=IDB{JeIf*KCfLg_MsVCXq?%ygXI(uFzg!Vj^Q=V+s8oOvy6
zN!9nUG&k_|23`T~AkJcmqUAR*qbSz{xPY!>Z4fX8L3o3|C*BpyVgsYLc$3@Tn#M&~
zo*TK4@`<*H7^w_U6`GRoquvFQknl@^)J|y}NgmZJ`x?nSGQSpmjK5(|*_kE~TN=RJ
zpG%{(E(5vryUxaqKe0ZNcw|3u3Q^i8{5F*e!0N<!3TtHp_Ow;9M~S>J_bsG#4lIgq
z2nGMMrvA#9{|XW${&~p;yY$MJ;0&sjn>#v;<xjEA)0@<Mf+nQUK7S}w=qM-~Dk{VV
zpqnq9F@MVbDjt}3>k3fc4@z#a_2Y^sFSv(np+l{XkU7eAucwJBLX?F!6q%$-*#=gR
z-=8PU3*owToyyx(-9YF3TloRLlRrY!v*pjJPfk{r46*^DpKA6jlD{MwLMsC@;tX%E
zse@cn87}Y8tc&<U4G-8VZ~21LI*@0wnI7T)VI^cdo2+p_-ql;(A^VhK<SjG@c-00P
zhTK4bNiF@Y^UKI4pYr3)@7#Qk@APa+RKzU4U%5V|%fpoVZQp-z7^L-#?7_bLH8r&6
z9Ac<MubsJs-X>l}0qzLr1NSXSvYxTCqd5c=^%2IJr{qoQ=_7Phn2e6Ll3X)~K5DOZ
z6ohIEXWLH(l5SC6M&q?>Rh0~~AeF>Z@+~yg(p<f;rT-K0uA=bYs;dCa1G)@Qa}r&k
zcyG+qh)~Y7kj@|F@<8c|!h7?zI<;+lR975VNa?UhQnfX6#@~xm{C_vAR`h|q?({0>
bdVTUY2p@`eTi@a$D5ovZ2Lb6n;WqyZO4J0)

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2exp_models/__pycache__/audio2exp.cpython-38.pyc b/animation/SadTalker/src/audio2exp_models/__pycache__/audio2exp.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e28abe8401f16067cd5c2eee501ab0780e77ebad
GIT binary patch
literal 1294
zcmZ`&OK;pZ5avr-t*qTV`ZQ4VmI7O#Irz{5K@g;Hnw)ABNOBRjB@~gnmZQg-<l3&m
zJvl{Az4RB@$NVLtx1xvr3%#T>TGw`gb_9+_9L}eY!;ddVqm+O($M^J~f{;J(<EFtJ
zc?`pT4Neh7YtoRMP>fkka|&->(}v}Yl4nE-6}=%U((Hhr@mwhOoJ`_1)RGAe!X2t+
z4+7S<CC=S6{1ck}0-Pp@oHG@_B{Qm+;%|mI-=OE>EfFM-0C^CG-!4_v?R<B#cv?n%
z{l;O;FNggS0!JM6<cyuO3t;b=Cz5*(UU+&Qd42&|87l&)bPs?dyIg2vkX1;etlG*+
z8F)FHhgj*Os?>qGE{ImUXOk!htLvG;UYY^A1nWLp7`-Trmd+Fa(9UJuS^JKLp_FA^
zSS#f_^4Cwh52i1zHg<Zjw62;y>5kjFE0mq?b`2m{tJU;*p<WdAYi+)20P6a4YE3yU
z`X1<$g={*dYkNI$fAO=4feLc6$cfM~jo~@Hd)2u6|Lm@0xZu)8@W!fL81{E?o+`Rc
zR?M>lIC_Y9A90ApoU9^89aGE;CC<2t&e#+3<8Q0jGY|;*h56XWN_+y*3~L8~7H^}7
zVCKpXAPac^S&=|R@;|wuADnYFR9vNM>q4wjPtJu;ed31~e6{5xzcr`d0LQ}}mAU93
z_7SWd{mLr3%6tY#KHdvy)vD@9HqbvqVZZC%;fbE6R}Fp;xY4uqW0=hoQ@E;=bGLpE
zw*CQYZi5SBJc+W?vzdhBQz-o6q_VxJ_6txI_BlkRXy-bJqe>t5!U>V<Ot}vlE8Jvc
zw)#b{3RmvuLtq6GTr6E+uWX<%%r<6emES3s#^}}|@68JH;Uzkd%j2H;N&W#YlmF(G
zXP~WR?W|N)>C7!C+qixc!fESJ1om@qge5ekDHH5t_9@??pRwC)%re19_FnMQ@wJ$+
zY(k$-V!SnN(P+3esYGEeYm74~50^!~$@JPZ_W(bPF0VwX>srbiY><ONO8s3#F<ZQp
s2_47%%p~gjX^^JDD<9(T`=9mM8$kUA`@JO(H*_B15nF`JXhJjoH^`kn_W%F@

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2exp_models/__pycache__/networks.cpython-38.pyc b/animation/SadTalker/src/audio2exp_models/__pycache__/networks.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..623697a0040fa70b46b53c006cc4293c58de68e1
GIT binary patch
literal 2149
zcmZ`)OK%)S5bmD$&U$U*D8UIJ0Rof(#R4BdLI?$nf`q^a)&Xg?Xf&B_JF}jh*>umw
zi8Ypxf|OJE1suFb|E0ci+7ovUz*n;mZ>-R(uI{RS)mGK_&HYX%WN5>a_vNI`*q@}V
zHV4XGbiEHDnBY0<%|<@7EXLW;?m3(+R?dc2&*JP`CT!vS#)KoSf&0wvd9b*`gT<R$
ze8Iac@~60y?MK{LMUm2cwb9Ge^al`<^|)X?OZd;(A#d()_3YWLJ?A-dSkJ|cf${dr
z;_<GycaR3n>6xJa*h1ex*WW=@>=_&II;dN}bIvO3+(vh5UR$+YJGDD;p}Vy;;S;N7
zu?NkcFwBD5gKX74WGGmdUGV*=ZCpJbNo9gK&WfyxV`bqH6L#gJu`H@A$(2op_SZ?3
zKHM+Wa95Zxg{$#{yiAXc^P~L!zOj?EGJ%ph6XPT|{#eTxrYJOymieKv)2uK~T8=Bz
zK9;JGd91TjX*^x2Oi1I85+SnUNUh;9)u9)iL>+149iPy6!A&cU(>&2SjxX5XzwCY9
zzptgz{lQpQS^t~zq{z!e=>A?g9KjDQMZcSfgCxgucZNw;T&jJo(ta`)S-C5pjN)M_
zWUl*#tWHXGtanCFl?Ml?>rDv8*SW(tdGL?tgxuq&8?#L9EJBu2$QF&FMSTxle+*Hx
zb5`>SKSMeOTx}y!{4>^Qmb#5qo08RKwbey2cA28`i5x{)zYK>$iL=O7>lh4wVr+y`
zZPS)U+$6JqSSmzIkkgt1$vF8NSRN0p)AhO6^G%U;@8H6tycfw(#<2-wio`gl`C1%5
zLLO%;O*GV-^g?HzgH+}@O6BtX8(3B3L{lv;CYt_6z}LgZl0tc@Eiv*T%u9@LVg13E
zI9YoIPJFY`&4!~~{;ZM_3iqd7LZPQ*HC9E+SKgrcb#zLGz5$JpbIvC~euElE;5Fcp
z<LA6^D+k&_2j*XR|6}oSCRm>F1?hCh;>@coE$pFPvv*p8^wl%3vV_>vUY-dB>8oc#
zvDR2R^IO-T(|V&%IzTxwfhX35-!Oq+S%cubg(CC^q1c#!@2K3O&HX8bYWg+_3KpvD
zH4JKt#5ITmRO8E*rY@&<u@+g1SWdkMG5wVU;=8a;2^blJRP;pl6=p}6G%ac?s*_oD
z(MIKyAP-EZL1!$B6xdY8`#vjV0+Jdc=U|CXj6-B)THqmFB}Ek*=Rua}h&T99O*JlY
z(Y){muGQouVS>=S#kZ_068h#}-aG5|>GqWxyM+3pZs{iSmsnyCT~`nbsGswipIZW;
z7uLi&12+TYs}`rWuUKrHbt-q@fyuFYMD^ml05P1owM!H+z?bJHNCKmM6W^?fedg63
z^5^b1!JYn#)}#S=utheE0`(DmGS1^no`8mlZhbK0!vwIoLo04os`MeaIjQg&AbaG`
z0(gs-Z6yzltCA`!du`0)`C-GKDhlTc{2SyGLo}h7QO|Z0;#>R{KYjgbIOiOs-g^0(
zOBQ(-3*^L{MYe7$vfwX$ilKL{aeSvJb`Hly3jYdxrl^EWcxf>Fp7DNY7_BL)=7trY
Zm-S(SFJ{E|B^9ZruWta~*Q~AU{{g%><4yno

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2exp_models/audio2exp.py b/animation/SadTalker/src/audio2exp_models/audio2exp.py
new file mode 100644
index 0000000..9e79a92
--- /dev/null
+++ b/animation/SadTalker/src/audio2exp_models/audio2exp.py
@@ -0,0 +1,41 @@
+from tqdm import tqdm
+import torch
+from torch import nn
+
+
+class Audio2Exp(nn.Module):
+    def __init__(self, netG, cfg, device, prepare_training_loss=False):
+        super(Audio2Exp, self).__init__()
+        self.cfg = cfg
+        self.device = device
+        self.netG = netG.to(device)
+
+    def test(self, batch):
+
+        mel_input = batch['indiv_mels']                         # bs T 1 80 16
+        bs = mel_input.shape[0]
+        T = mel_input.shape[1]
+
+        exp_coeff_pred = []
+
+        for i in tqdm(range(0, T, 10),'audio2exp:'): # every 10 frames
+            
+            current_mel_input = mel_input[:,i:i+10]
+
+            #ref = batch['ref'][:, :, :64].repeat((1,current_mel_input.shape[1],1))           #bs T 64
+            ref = batch['ref'][:, :, :64][:, i:i+10]
+            ratio = batch['ratio_gt'][:, i:i+10]                               #bs T
+
+            audiox = current_mel_input.view(-1, 1, 80, 16)                  # bs*T 1 80 16
+
+            curr_exp_coeff_pred  = self.netG(audiox, ref, ratio)         # bs T 64 
+
+            exp_coeff_pred += [curr_exp_coeff_pred]
+
+        # BS x T x 64
+        results_dict = {
+            'exp_coeff_pred': torch.cat(exp_coeff_pred, axis=1)
+            }
+        return results_dict
+
+
diff --git a/animation/SadTalker/src/audio2exp_models/networks.py b/animation/SadTalker/src/audio2exp_models/networks.py
new file mode 100644
index 0000000..f052e18
--- /dev/null
+++ b/animation/SadTalker/src/audio2exp_models/networks.py
@@ -0,0 +1,74 @@
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+class Conv2d(nn.Module):
+    def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, use_act = True, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.conv_block = nn.Sequential(
+                            nn.Conv2d(cin, cout, kernel_size, stride, padding),
+                            nn.BatchNorm2d(cout)
+                            )
+        self.act = nn.ReLU()
+        self.residual = residual
+        self.use_act = use_act
+
+    def forward(self, x):
+        out = self.conv_block(x)
+        if self.residual:
+            out += x
+        
+        if self.use_act:
+            return self.act(out)
+        else:
+            return out
+
+class SimpleWrapperV2(nn.Module):
+    def __init__(self) -> None:
+        super().__init__()
+        self.audio_encoder = nn.Sequential(
+            Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
+            Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
+            Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
+
+            Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1),
+            Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
+            Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
+
+            Conv2d(64, 128, kernel_size=3, stride=3, padding=1),
+            Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
+            Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
+
+            Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1),
+            Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True),
+
+            Conv2d(256, 512, kernel_size=3, stride=1, padding=0),
+            Conv2d(512, 512, kernel_size=1, stride=1, padding=0),
+            )
+
+        #### load the pre-trained audio_encoder 
+        #self.audio_encoder = self.audio_encoder.to(device)  
+        '''
+        wav2lip_state_dict = torch.load('/apdcephfs_cq2/share_1290939/wenxuazhang/checkpoints/wav2lip.pth')['state_dict']
+        state_dict = self.audio_encoder.state_dict()
+
+        for k,v in wav2lip_state_dict.items():
+            if 'audio_encoder' in k:
+                print('init:', k)
+                state_dict[k.replace('module.audio_encoder.', '')] = v
+        self.audio_encoder.load_state_dict(state_dict)
+        '''
+
+        self.mapping1 = nn.Linear(512+64+1, 64)
+        #self.mapping2 = nn.Linear(30, 64)
+        #nn.init.constant_(self.mapping1.weight, 0.)
+        nn.init.constant_(self.mapping1.bias, 0.)
+
+    def forward(self, x, ref, ratio):
+        x = self.audio_encoder(x).view(x.size(0), -1)
+        ref_reshape = ref.reshape(x.size(0), -1)
+        ratio = ratio.reshape(x.size(0), -1)
+        
+        y = self.mapping1(torch.cat([x, ref_reshape, ratio], dim=1)) 
+        out = y.reshape(ref.shape[0], ref.shape[1], -1) #+ ref # resudial
+        return out
diff --git a/animation/SadTalker/src/audio2pose_models/__pycache__/audio2pose.cpython-38.pyc b/animation/SadTalker/src/audio2pose_models/__pycache__/audio2pose.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2be515b59c512876ab7e4998e6093e49382460db
GIT binary patch
literal 2866
zcma)8TW=f36`t8Umlsj8;~P-7P1+*NMF`xNpfHLUmJ$?j)D&|1uu!pG?+odsmrH7P
zDL29pC?E%U3ecZGJ?1a;r_5`M0{#nmQNJ^!CE7^RlGrn6ZfDM%`Q|(GWw#qM{3g$T
zE&ttP>|a#b{CTK+g0DV+5KQorB~&^koA88l(%sUTxQT0Zuk<E<;#=J>gGrc#oPEYb
zAi@_+gp!Y=GdF1oe#i#VOW0%w18$tEg7hB#`N2NQ+h0s;c_@$PvdZONQRk|d6jjzt
z6*@Z)=AxMHSNT*(m6L(ZAGu4vdKW^ngbS89BKn36xp0L0qLsM96aEX9cq^9@{~PA9
zB!D|D6I<K&XnEu1bCKn*?2%0$pS%8Q1*KtUY{}0VXPRr4I+~B&u{ZXi2Wvfq9%&b{
z)x_G-?wJF<t=YJ}?AV#4b<f!z`%Ap=1lL~LJ3eSU&Ckf@R?}Z=+k(MI=it-9j`8dH
zOezzlX;BqTni@ABjuQX#Kkn^6QsiP19`64seYAgI!df1urL2<nqX&=o4<4t7e@OOC
zTxN}|np6}M69{=y<WkZ0OgE$GrU*QR*Ld<ID@{DhR5p=~RJG|UoJ*l(osLu{j91C#
zGeuEK+EDk>$+Rh^)xbAiEz6-Ia3wp>vy*#eF-!BOGJiIk7FDCVw3=@Xf5%NLP4hCV
z>oon2egD^ozdQP}#$rd~dEFF8d(-DtIn6|U^l&<vVd+|mqr*%*&dO&}eK^UA>Z*EF
ztNh4r;@%9$2!Dhu>y4J(*;lHC#Up(64G6|#9`FwSfzyHB<B=2d(`#2)*nJh}+@bO7
zA8n0{Jj7SO#~CZw0uT`VHe0fD07E$EeBo*rH-vo+@L37)eeDe0+w8I(_XoX~?YEeA
zL~#GAf1?jQA$quou!glyE1@;At*zHq_C{<0C@vlCz2?En8!Lw%X1F3g=kPSp0W7s)
zi7h=%zB{OQZnH&r>FXPNSo-H|0eG+Nzp)b?-eilGZox_yRwI->ln!9Lbzq!PqYyOq
z(uz+=`xGMSt&fdhl#UwXDmgU1MTd!tN)#vQ1Sk5Ej>5FoaGg9c{mpqmi7+kOV%5;9
zqh=7OkmiLo$VHw?1<He!dG)HkLv^e+t>hq9*Qmz-X8b2vlRr(Od@6@SIHi709lguN
z)3T@=^-DCWw@Lhp#5*KzUW!}LOgNmX=a~{zseb{%9FOntkNF*bgYUQ=|A0q4b|QY4
z--V2v)13_%t|z~SLUNIiXi@kCLAgR9uxHV@KAklJE%qG2En;x~I28}NI|irYtU)}&
zN9k*SJue{K4yg~MCPKnP!*zJZMR1F-0g4+tcL)boyMZ$Y^5O<m2-|cUd=Z@UF|CB!
zm{3M#D57uO5nr^MNJnQJ1o-zwr*W4p?ZIMv?kwBzDg+L@x{En2c*&L>4S&cZ-Jx}_
zGu;uvP4tkCd$fV>l|IC7V56sduo07uad!#b;=8@vLTj5waCZDp-4f0-_aBaW`~UYt
zto^XJ`zKaMd;>%qeqsf#{zIDXty&SsE#15HJy^wi*%yJ{LhRf{AO39XzV3~{4d>4P
z+RfThN4KtTctj^p`^3p?>o(F~_giPNt+y!xgn-TUS8QdmvW;83qjxas>h`Li0dn^G
z7T;f7T&TNLrAwjiK$y6iPtqafjC$Z0Uo>AKU0KHGr&rhHC1zeAqr1pL_O{r}_B5MG
z<5G%flPMorKYwa`+|_BVenU%qNMe;gR920o@ll9Mb!fbU@pP$`y~$o9>&67xY$hvV
z0wrfM1Eu6yGq`4Nw|bYx4oJ|JjPs<n`$&3MRFlq;j6R@VczR+SCDnVVC2zgSIF<$s
zFBCmK>^_rlmI=zTNtcvM!dTJ74={xK7*jvOS3iOvb|5Z5S^^tfhu(2xq$m%35nV)X
z2dokC`_#_+NLp082R`XzPT#h^>rgr)E;-%WU=j+;fLkscg!GQ6a8`In#3?1rc}aRF
zO^@eUx$3bgQ@szDtcRw%PT09D%QW4fxwmKt(VGe&h-_<8BHrsqw`?1D^|#bd1fJYM
zy56;6*-Y2F`AH^M51Ds<c!1b^6|UYNKQeM1^VNf9^~ec6w|lhSY`AW@+R}7%e-(w_
QQ*NqlqPGtOJQAG$0&2v%qW}N^

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2pose_models/__pycache__/audio_encoder.cpython-38.pyc b/animation/SadTalker/src/audio2pose_models/__pycache__/audio_encoder.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b6b644ceac2c77b63d3abde678a3345c60516dc4
GIT binary patch
literal 2176
zcmZ`)Pj4JG6t_Kpce0y=kOBn`;a}A`G^&CGQWZs|34sITK-$Y{(P%pMCX>w0Z0y+t
zRuc&+K`n<O@eOv5`3QU!UpeiGdk?^Sp51gQ!IIzjJ=@Rrd%yRaC#_aMp!Lo^5x>Le
z4^-xx4a!64>T3|3aGH@eepyC(v`s0Ft&H`owoS=bgxlP?B-|17$holEE?8XdfyJ9z
zJWlsX=uf~K*$b&=c@8q@4)UZ-i#*N}oSAQU18nLJ2tnGElQ!e-Gt#A;aqF_tw(3*c
z_A_FWwgdh>?QR$O=?33TJhQnL4ftSRg06Nzl;naO(aNj*U*Rbw8(GlpidL+$D!X#V
z4(LwB#&pao61kvxV*(?;azQpY*(X<YFATJ!27Mv5A4O@NmQf@bxYxnHI2nk%Oyf-A
zJM_Y%xJ(ZB3fbG>I!NG-(bKF*j<x-r_~wbW;-u8R6e{I|I13xvRwC=Rtt8Fcc2W#V
zy>KjKF0x3aL!n(&%9IQ3^<&P{{6Mzw3_F$wO1sBr_`RaK5k*NBD-}go<Ylt`pz}lt
zsX9jkRi>St;w;aKn5)iq(d)y>O7PBp%pb=YoOG`jr}-<jqh!*F2Rtn{`h^lvuizq6
zwIvdH0;*i^|0pr$E_AgBg3v{3(`D-a<=O!qF4c)yp9LzxK+T#31MJ(-)rTM|G9ncn
z({l*_5tZ*iV(8}twGqR#>=Wt0ve1$UWAlF5tPOa!rax#4!Y1FsStH&?v)3&ogohg-
z_+TM4pf2_4aB+IQ>4K2%Ub&dnJno4o(m{le9%MLfM$rjmsh%-!mhYnT)->LU$T9%J
ztD{$8R^s^v3wUpZpz!Hp!&v?UeGCOQ^)ZMU^bPO<`j<WcAE19p=78_$<`<ZsFY9C^
zGY{wm$G4_m^%4dpDQQJ6*och@px+wVrNd28fu!7i;egJt$c*lA_l5gE77y0=*LysU
zI{KLHxyjOiHMpuir@>L5-*b~CfHe!(djgL7{GNa}jioj9+5{cvjXrK+4P(uQi+d9a
z5tyu@z`y{45Dan|#Ty_VhxTh0gRaR#m<yTw2-kf8V)7FTh~LaQK|rSjq{JgaPhu&^
zH56DFh8ZM+uBL>RDX3Zbz&#9bSI*+ojV$d)$)QM&`$d|U+U4RjO@uT!x(2gxikeyj
z0oBH^+E(ZadrgAA{O9o2JU(u&LPRFs6yu@}K|F^l<faNsdc>}o*DozIV~^<t-60ny
z5ii&;She<Z0#$2O){(>Q+vMD?tdUy*Z&x<3wcT~<*^$THF|8%?_MVAZf`OYg3@6^`
zP3Y=<5C`D@`_*&$9REG!XJCXbyG?#$C-#&LOv!~zv-SI$roezpb%>QA<3y-ACI~jO
zRFw&E-_`>>01MKE<PGosD^627Sl<M)jx)6dgV{EN0{DknJnc>Xh8Frj+ept&S;*v2
zTS;7MN5=Vqkaxj7{3{OQKCm{hZ9P!%lMq7@IwtpW9T(9@?di2e@2RNi{XR_8cfW@t
zLnTN#pFvkhdxQn-4hw+HR@rd%rWeTfVAYMBv<4GM4^vKRt{KRNzWfw-U>3A*9<-k4
o5}~2#u6DmQY-o^NL;MxpT_aTcn>A@}VO|wN-=|I5WGl=60Qwx}y8r+H

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2pose_models/__pycache__/cvae.cpython-38.pyc b/animation/SadTalker/src/audio2pose_models/__pycache__/cvae.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..90306be36db3f2bcad9426aa657478c6f5b9c61c
GIT binary patch
literal 4699
zcmbW5OLH5?5yxj|u}=UbL{enrC=T<ogv4eP<>8bfKPVD*l#rAxl1U6v*{m>23J@OD
z>_U>^WmPJa%O~G*%mFwmA0Xw=GN)AK<a;UyU7UZ<E&!5NQmHI!8#6tRo!RdBbuV|z
zWuM{dJo#Sq`4VIQpvuXsfXchL;{-x5VYFF2`!zb8GlO-^dchzKZ?g`sb6qyuR>!W}
zC>MnFjMbg@nXrZPj0q>=E%(^0d(gPTgT~7>J~X~4LQ~8&MPY2SYUv9+nQc`K#k*Zd
z=S~#wcB7=hNuOUPel6T<xZ@iT2^$(I8yQ1BVyW?v#fypgCj;6-%7&IOMn-Ck_|Q&S
z%NZF*Jmo`I@D$q^nFdQa9&Bz&12cU=qUtFt>B+_eWp=`+s?bcLj+$AG<47jWUbiM?
zPbzB=$zH4qadQwUp7fM0qQ}ifq};d>CP^fNs;SJTcv|-w?J$mmZogBvd)??kFL}=7
zES~eCzIJVI7qiBDt$v&|_ttw)y6s*l;=Q$A=TJs?qS)IG#ZK6M7|Ay}VY54}?!~gP
z7xqQ7clEFrM?t42qISI3cpOG6haahue(Kfifi|8NyAX^o8m|~LhCGLIgA^w(da-oH
zKY@r?-4LwKKV|!dFa&?*)=goG0xS$W3md<daD<Cr`%_kc-SHYdRapDYYVDM{JWS@`
zrrF~^LL}@M1{=YjQlrINW~%^`WF_m!%uDuB0VSv99$C4@JF@eVf8^w);*p!e$U!M7
zryM0KwNoc`k74RyCYepmGrEdkSf#O5J*TX=e;7&S20^piOoBjJo43|$8*&aStk2iB
z)=;|>Y^;6=xqY)<+pf>A*S?}u#f{aS+SZQlQg_x@cUHG+JN4q$?q;yIvAVsDapzWd
z*KghmYMVELn_F4OxvA>bZZ=ps+qDmZjoOwfw!<XqCV_S>CtrzjqHY5wmdeSOp&Suu
z4ZZa^YVRwvv42pX&-(@K@FRF3O%m7V^R6eA@@v?sD#9-V?JH5N&S`%NqRxH1l<fI)
zQ&q1&UQ--B3ffWkxuLuu8yp1MrDX+k$_0oS-0^D=%<%I+H~Zx_cMZ<)_vN%SD3jl;
zjE&J4yCO|MXPwZrhCBW)1f6$bq<jHq{>Vg)rzT2p7?|5!U`u3VzIA}Exc>f5)s!@c
zGNY%5%9mkRbc1&DVN|u`H%QMrePy?M2aiLk%s3I~co=8<p>wJ?rDkMw7|O5{;cVlb
zzKVvJ&K1LP@{Pf~UjOMnr;fp?UHly5z$P33tQ&L^#sx-)Kydt;Xu<BVCyqT~SHiqb
zz0p1*NEWu_C5)GsA(Sz|?p~ql4@g{u7;nqEA0~|l)4uX`>Y;Zu-IlZ8lTSh^zJs>7
z1VQ$4@mIn<DCu40T~2L>fJ{cxMxn4W-VU9w!Jqx*{L0|h6w7Zd>*d_Nq&1iBO?nK_
zq;b2VEpz$qc*qL*?&T}h*O$q=S5BOYd)1YscX`q;BX4ywWB#DQrF)oqFi}^H?DcSM
zb@>JsefDh>k|<8-MaDYd*aJ_$0Mb@nVnr7Y09N<`@wVTlaw!NN^}}}7qJ2Lz#R|>;
zLkLyQ4@o0xx5o!XTYz4Ac4UY;W!#$5c=;9Aa$U`$2DGOxdh2xQqwwD-t*LCm9+td-
z{tbfg_#%r<p!`o%rch4!F^n0QfF}zI04fWt#2z|`tt@cKS1on{Q4=i|oi)Li*qiW$
z>n!mSAMH$-pTLh%DhNiUV}5L`vwQ1%{K&!W9(jnwZmTG)VJSsK#_Z+PO}!Bp_Gd^c
z@>XZkIx|8_K`oE&>%KS<w1N@Gie|r}sb~#?fo?yC+FUX}T)?aa8iQ<r1?P1jaPz27
z=lF{+aY^FL3v1)!YvSlRR|PVvvNKb^uvST3U6k|alo^_Ap&Uf&K9UsqP11jx1bL70
zx1&dWfV>&DmAScbTX_*+8%fYznS&-!6bh01k+N~fI$@&hjb=9jhYEM1ja^l0>sru>
z!fsqI=d~PF<3>$r<=r0BDTAG*Pm{{BkNeFquKF465MCyMLC1h>-OIG@Z;_xKD5I&I
zaSU+B<d@oEC-BhkfO3U+8G>;iV7CpM`^H7%`zX202jVaCIb+V4;e$#Z%BGaphm3~e
zqJTk+T*DoI4ndxFii+CmAfnkqs)-gP2ExKGJQTb%hwz>e@zhD3p_w8%0Y9At9^7Kb
z26@#G=>gi^)Sa}u@CrL6>Us&14bT?pL+TwAD7E>FANd&Nr9MU#QS#FwN~KAuoEpPo
zT1v~p`wRSv5*i#ge@j`KO=>33%qVRP4iaU`XkX`<%A15$?Q-&`*f7GWDVm)spCY?j
zLZ0#<M56KwbWmQu8^fg{q@3h*)S#?G+3Y5=hIaW&67&+03WSM#i>m8XwflzxZdaX|
zqEESFe#$NCcZ>S*`>}Gfr8T?C)yqdVFkUYFK>67)@+eh!iabGCPXjbe7kvh*l0bM9
zcTA4Oba0_mz%?sI$)x0<1dmxXd`O>HjKS>KPx83)+?H1`Q2vO-RT7k7=xODTsiYn9
z3{Z)9)ELu=22jsu3`FJB_5Z;1>zLsbOp8!VVEQjq{%4p@A!;l!I4c3vIiQ39SS55W
z!KHG1wg{cLJ#tEoPZNN2b9|DMa#7IubcMCVk{OJ|$B_JM-rJt^o&`>I&qq6Xn*&Hz
zrdk3ha(;X?0hI*MGUd^_Hiz1LvM@Z4kyD&5j&Vvf(9SG>Mx{4NkOky52<61hgHEq0
zs!L}Ws@?kxKjn2Ar+xTsD!oJET@tG#Za^p#S*RppAbZZHRz>Z!87Z?<@g8-jQz73c
zL5HMT%nky%<$q?N-|QadIj{T`_53x74H7Q_p@2Sf@e?4d7?&p?tQZ&hBA?MH%yamc
zqmUAT2@3y)$^?ZR=p$I+D~UoNrMxdeAwiBHh!j%T2LKaltc7nOjBpXSUHBSL3M~uY
zOC0`&Ucv$F5SH<g?4iG(`ZOA&9AWQMa!nv~q4fkj4)q*|!(v`90fi;urKKiXCm_sH
zPWq^pn<NM_@-_(qp8S9WxreHpBqct5MYSxqsa11rmr4`3$=^~{I|dOY^PL?kMG$bZ
z43YA8s9_pP@v~%4=f8ETen>(u<}Q`)k@!7{fW#k2?2$MiF~y7ix*?o>j5`l{8f64E
z(}!&-1TFw+bA<}8pgx!#TRMlC3{Ov~9ZAotu|o6a#9ET1sg|_s*5|Vnx!YaY?{^y%
zP{MYeW=s{zc(48{{x7hi5q|Rj0xJj?flgA0c>42S*_%3r)mf&dqkiCt?dpxw+LU`e
Y%Yoh|Q^pjkUBvVf$n83FugzWgH}s7tcmMzZ

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2pose_models/__pycache__/discriminator.cpython-38.pyc b/animation/SadTalker/src/audio2pose_models/__pycache__/discriminator.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c993870aeb1798c688d1f0a9eaa636dc787eb2a3
GIT binary patch
literal 2464
zcmaJ?&2JPp6t_K}JG%)XKt(wfRA@)gwB*pL6tzG;O4Kl*d>p!3X*7(TWHz(28+(>O
zH0hy4;*>w2J?$QI??2HV_yc_HDYu?cCHmg%F1t}nJ@T{P&-Sz5=WI?*c?4Q(|GH?w
z>`!DSn+eQ4=qdoh38yKk4@TNzj8M`t>Lx{mrKH8`jFRsOH@JC3xG7lEdTG>cKrC(p
zVh<6A({<uIry!FAKGiJCVw_L5%8*evU?NDJa#Cl!`ig8(?r`_0Qa5;odq<>hz9J^6
zTOhHfYYXk{S<sfPHIa5TyUkAxIDUE$$a0|BgCH*T1r{65UFf(+!P7}jUXmW|gI<Nj
z$*wDJ6{ft#X~Q_Q4(&rH^EeyL3(AdMCV$FV(<oSB6josu4mY`VY!@Un3zyrl{xoMj
zt1$W$FnjMcpbntV0d>Y`?*2@(Cm_*<wbvlA@;?%PgR|b~Y<6WR@ee?&?@$vZf}C1!
zsvu3XIo+qAQ$_v-*g3G9$-QHidBDx^Dz6>m>&|lmHk=B4;}qR~%8-Cjh;K*>28re>
ziDvVB8T@8GO(G>7fHhkQB%a#-^%rcvf9Eqjor&!zPo9ZTrR|Q;PLg%fC>9$&mFHlu
zKdUX(?FgyeFif%}4?~HUY8$;c&$SuDsH+d6Jl;X4B9y_Xr&ltSM_DX}=!Kx2ue3U;
zXo)Nrd`Zf-97ws%C(Z+N<|ObY-mA~;MoBBlwnLO2rJ=U>MY6q<>&mK#c3-TC)lF@d
z)QIBT_q3@*x>c{>`@;MM=vGa#Fy4u>Or%QJ+TDE2dAz-!sR*B^tGhzNmQW=JLfa~r
z2?x7$BF^D<c?p%bQoQnn>Pi^KX{1ybo{%^3!kxyZ5>hpqU6m(|MUa`cBd!_??N&z$
zr37!RM|>kn;fl{%QId_>hLUk3>hh#L-)Somw%Qz?+2DzaCF&jJZ8_I@A<x78HgvTN
zgfNe~)TA}$F&x?Wd*iw;orcwEW|w>Zd(EUX?BLR5M$V1=9HXB{4d)nO3((a=AO-1>
zg7)bl%^@(GEH|3wOD4YnoE5ZZaaijQ$3vQV1046m%pRO9u*5J3W{Dvzv7D5#>pSuS
z%(a)OWEe!H-OI?6S8&OJkZg5xAwL4{gvqN2(&zFfj018D5ofC{_an(~1EXp{2=%}!
zm%(lxJ-B#A@8MRzQYOhDozT+DK@gM)w0vS?7GzONgrW9A^iem(c`Xe0x=}h<DP1J-
zMm;qQk65HBB+mG8KCDW7p~Sj|hE+2_Tnc4v)ZR|tELY<ZFdFd)HuW)(QQ0kvF6HdV
zg`!&)_lcrw=?|YkP_2u-u7HeOoD8ZnZbs2l&~a4HflO3etUnB0-2_)RtwWaEJtucL
zEog~UIO9-dxp{0M3Y>L@hyM%y6}Ss~D#uQ4H$6b0)*=Q^_Kxko33pV>8dn|Es#7$`
zhR-DGDnA915QxPDekWvH@qANW$94P<$xlW*|BYe<_%(^9F1@6Ucxzi*j~*{Bt=6X&
zSJoHSRvxVc_ctD|)xFiF`#<~`t}U%@4l15EFkhZTY3Y~x^h)q#b0b`Mcs~f1R@dca
z5UUL(_8_1!9_80a@XfOHcj^Ap>6pZKp{rRS7?L)mut`6lGxQ?8`1irJ_j>F7*@>aL
zfHLm@fege>F$>F)_2F08H+l@*V)j{?g~ligdnSi<z%U=JV33XYCpmo!t;|c`(hgLx
zoe1ivSyskME~89gt&*sUb`%7cI1uu4;3h)7R2Z`XtA=uc5PA(%hWxwwel>^5DCc4O
zgs^emd<6^NKvxh5kT<oOM8tQs1%)!+sb46Exh$L8>Sl4Cv@>v9`RDS+fTqjZepF_0
e>C#fh#C7jHrro=PWd9cPv(&<+v*5JZFaH4qc1{KW

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2pose_models/__pycache__/networks.cpython-38.pyc b/animation/SadTalker/src/audio2pose_models/__pycache__/networks.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..45ac321062bc5b2ba8f2a2f2e7d155ea0eb02c9c
GIT binary patch
literal 4751
zcmb7HTW=f36`t7}mlP%2tSURM(|g*mXk0`s`j7@~WH~NS*s2lPKufUMu69<^O5~EB
zSzSU<eX)U{4+8quSMgDQX<qxJe^9hZlkdzdMN+1cW{4fmoSm7SIp;gyIrDsRvBprk
zM{lM7yvf+Vs54zWbUwhNUZE09@DA(p7U#C@w4C2F;RyE?6K=})+?Q_4gTxa)Bz`IJ
zAqhkUl1eEFAPGekl4>cbKvEM6kSvsv5R$rBgk-UlR0ZE;&7}+Mgl!~LPZziBs2`(|
zvKAMt<$TJvx!}Ti6}DXA3hxzbd7m<mwS27P>-tBj%EUq3S?}eCi9fX*wHNWws_Jj3
z3RdtTd+D4xBgR?bo_TouAs=!42O~)KDnmYW1RuIXPdKN(dGE};cj!Is<J21kLl%X0
zoyknRo0>UtFs3RP?RDdVw$gO<!hS0Av=gapkm^7cG83t;^kX5ieCLAGAV-qfWI*kj
z_SHc@mC`{^hf$Q}SrJ9r$#Y%XOkW(Nd6C5(z3?C|lHHA-?5+y!eUv_at^?8vdbI@q
z(N?FI?CYv&tNpAG^?54oZ0l;4_YaClWL;hB9Ta6-k_OEd&bh8eQPPQ(ilTG&pO4qy
zYd=@1RPDV3Rb=gly`#L-i-l^h_qu(VDwT@%W-Ok?9US$YZk*+lZd=Ku9Uq9Sx7zQi
zH0t(5+EML1EslC}U)}G2BCGf?9bNqd72`{e$3q@)k2|dVec`zzx%uy)=J5s(|2O#d
zG!X8Om6VXq#)aCX)^$8;8P$-TF|t4YIsM;;ozD3Nw0{F{CA^89JMtF(G(VOrBs0}*
zl2o>P@+g+#4m!$w#u+TnY@!@+9+1)UxSPUgwTKRU(4lrcie3P@_Ki8T{0^<WSn4`S
zJDn(+`Y!ER(#H)H$UKV96Y_e)ND_2r3IUGFH}R<Od0)ldekXm+o5&=zqA|-=Z`$+7
zcZ}!oQxB;34eA}zrol38w;9L~fAR-Zk(DKJm%c$TeJ0~v5h8%s@Sxm7(>Lmu4I29`
z8XH*WvEVccxipDc@w*tLmQY#TjPWwKH9pq$*r2nm({$R@7#o{Rl7-K?{1N`-8&u6A
z!5Hl=$Sn%^;O6D0N(9Jzv$H04Bt^VQQ%gHmjA{}{W(vKfVl+QyMiCfpB<`+)8x3Pm
zabra<Z-N`rK^lE{oMc5BJs|(TAXF!G)N!FU*@FI68`V|Qo_WGJNkoJglr#xr-!*~b
z)P*FZ&!2)wEfL;@TTxJ<cW2cR8xbIC1`xr9tl%4unp}HZS*)IcTq@w7PMoC81q%;y
z`zd(bS`%@<$PUxB!=0zSUWWh@AMQkbwB-WE>fmvfr?Hg8_qwu~?R0yY(9U+!bglbV
zlHE85=jy6V#X(YJy<E-#&(APlk%9moUI*bccnweR+2GB2x||Osaua&i@u)|rhWw0;
zSlc^sPTUjk1iG+vbB16tbVtsK4~aYUN8CyfVED*`ln?zgxYdX+6EQhfbB47MIpoi=
zwDu0O^ho}Kn$`5UALk;972J$SOWq|h-_ktMJktD=dBFV@#wc=1#&6=ND|~Qo?omr9
zH@W;N#$JzTvRxBTiRh)8BuRp2rd%TW*AQPK+JI)q-R1__glIyx$vj$~%{N-UiT(j1
zYE64<n@^u!kg_l8R1r-i<E7CybWh%Fu&vQ}9oxVxX_~@1tRdVtn5_}w3G0FIML@_t
zs|=9}jNGZ9KRwSe^H6`qylR;&OwX&Bd8of)UJdj7VqxgM#6Fi9_Uf2@)Pb8tN@k2_
zg&UHNY1~ViEW(|Ha2<-Q6BF*ANixp`MXctM^|$wE+9e$umy$%G#-Ai<RxiY=-?tgW
zyVIT3>CP{;+evdB*c<|J%w#G@X|}UlsHRuCVwO{;p>&90w&lsVNay+E`}nllKxN5%
zk^lrOKnhUf;eUgQH<cLtAM)@HA1o4XuJ>L-knvvn{x}evAeFOzdjbr=Q{k5<NFfpU
zOxW9T2|=gUn|<<TK-4XLmTMPhhF^Yw73CUL4^U~p=*eVPyGdNghY-wou5<jZc8*u^
zK(SEPJoo(*CYgA}z*O{G=7YL9UFon1Az@r*GN5OW(&0wzA^Xdt6OQbd7Y=R!j(i*c
z9_7Y+!3Z~n7h(CXf_EWF_rf#e^7Ib98hQ%^!xtd|QLbX)W?ffoqN8*zQ(3NxI8Ra&
z?+v!JN8r#m_T#LJv^UC;x5k}F2No_Cb)JZeSo_-<5+V6EmeoFpuNNVwuvuC6s!l${
zpa_p5)G}~*9kf;ln-3b+<};A2ND97^at8T5s*Dp*5+-RO`F*M;(S8!Q$zy(lM-lgg
zeC1wPktDmG=ZFO_dG6~XcXgaWMDlmEs!1==pfa|hp|l@DaFx>b5Uwfwp3>dJW?}Az
z4!5$736f1mdz5qQx=Lc?q1$l+5RNmmKV!iuDxmztBtBz}`)1NSU(GPokUsh9pmBME
z`EEw1pc5E3o0D%zN~o^QwsQKGQ3{_VcG3Mwj3TXFAEW4yyR-y3ddQc2Ns`xWNBg7l
zO|s{uwW6T=ST8r(*7d>YA6WRR<*vDr8e%~*MfX)ow7ijH5~?zsz)!E<Q7>nhFhSwN
z-gVu{T^8XCH&qHGX3T;KA^ubdL8K>e8vr)zHc)&_f-8Z79L)v_3K_ckBt9;aw7R)z
zrg;LiNo{)~wLN<ajf)~L<3elEUP8;^<8C?L<o@$er|?ZhUSK1HxfJr+XA~7o`|}{M
zt@`JK+q1Tr0CQ|;T81ET$^YcEWBxbn4@KW*CI3U2Xt@-T%S!^X&A(!XjH%i}Rh~^g
zCczV`{z%n?QOsqI_~!3;44E)6c#RKMuI%+|ZxY5dw1~xYeMOQ4IZ|js%>6vK-1`Se
zb@0U8LCvMnUdHI8hQW+0n7LEiG#fUz-tc|P>c6G1V(wXW9=eS?je4Wec&AbO5>Hdo
AzW@LL

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2pose_models/__pycache__/res_unet.cpython-38.pyc b/animation/SadTalker/src/audio2pose_models/__pycache__/res_unet.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..177f47d21ba047153d82d1078da3905bb98521d3
GIT binary patch
literal 1924
zcmZ`)OK&4Z5bo}IdTi%KAiDvU@Defti#W?`h0uy1aba^v$Yo?P8m2qRB=higk4ZL~
zNJtxLPx%KN?BiaM_%nUwl)r!wP&H#a4iRHjb#;IBn(C_Ye!J}vc+%7N#qTa5f8b{R
zF=6u<NIe3^38xA1>y@U=XB1_Y7^&%-spVUgd`Gy!%}c^f!H(_chVO8?M>^Iu*pr<O
z)2&^hVm=I$?L7NgyZZwbrh`O8h|fRPWTSc@2;x&te8wqf=$~7M<^g*_4k)*|bJ_3>
z?()VZ@y!>+B)$a^J?%iG{Y(_s6g*sSAbX7L7i0tuBLk%AgK`rHSM@8P1u1Arp0g1f
zgK}Yv%)&UfN?Jk|_u;KEk1IGE+4aeog6xbz6SOLw+nQsuB!T;Xe#Q_hXcXSe(tsEi
z#IUc>u2~^)2DozzX;lcE0m5z}?P8%mn;{l=D{qUH4QSWi)}~&KS-hpnw)R@bo7yhV
z#e;k*Tg>&wcBQhdb4A30Umf?Axwn8CR<>Bn8{5^oKBGwmE5sOMm0ul`F@s9H#tGbR
zV@FoEink@PRiLw)CqiZ-2~>P8v@-}fkF&!LyJmAbwkl-Ig>2zjySB4yJGp?{HEpTk
zKuC$5uiYSsv$zNX&9Y2;d*bO(WJMe%+Q#<naozkTETW^GT&A!wcg6Snx*2DKVG$(Z
zGa<ECZ`4UVkeG;Gl#~7sBJdDw{`a7#?VljULm~0qv^Sg#2xu-32f;1t%{jPt7w*l$
zTX*3t?d-*eX&!Uk%7?`)=Z>LGC6WW}L`Pu;71ho`oD@Q;msB@`AWA}|g5V|j^T+K+
z{e1;M|9Gg1xc_Z_nk9M2Rew8A2au`~yuTOnAHoD`{Bat_+0C}EWYiCbJkEQAT!|pf
zxkywWtb!p-ZFlfYE<*(zAk_s%=mIn8Ds54l-uuh8LH@@!*XcTgAAM&=CS9TS-{(ua
z_4IeAR~TXWrw?8|-tab%`VCk~Mx>-Tg(HgV8o7XO8?lm&>4gE3Q5qn@ERM|51jz!)
zDlL=>``9V1(mbrbPgm^12FxvOz#NborGrwX<&?%u%LU9UUBDV3HOmG{l~$u<Gc6CW
zR_OuO1gTv%L28vPm;qy_?tH2Z9;clJxd!hc?;*R7Y#rI_$ld^^ts<Avkv5{RkZ<DY
z17vR@+d%d<vUh=X+VVYEG(FSoY!j#lv<39gXJ?-ReFpS7&=){o`o`JjC)%zXNOsUk
z#XdwA&Ot6uL&-mYjly_@tx}JA%riW?!eE`}?0j*0X%oASjU$+aselpm0_>4tf^sVe
zp2B}fjZ}RiA7a4vq&*{%BtbBj4rU-Rr)scm5ES0OOV>B5la4Dtg1vUDDs{6=uE5rR
v08QGx)1+PK&(mC<sA{C^LA9S$xmC{Q;@;H4{i@FXYy5C2%)o_SUNZg#T$kGJ

literal 0
HcmV?d00001

diff --git a/animation/SadTalker/src/audio2pose_models/audio2pose.py b/animation/SadTalker/src/audio2pose_models/audio2pose.py
new file mode 100644
index 0000000..2b8cd14
--- /dev/null
+++ b/animation/SadTalker/src/audio2pose_models/audio2pose.py
@@ -0,0 +1,94 @@
+import torch
+from torch import nn
+from src.audio2pose_models.cvae import CVAE
+from src.audio2pose_models.discriminator import PoseSequenceDiscriminator
+from src.audio2pose_models.audio_encoder import AudioEncoder
+
+class Audio2Pose(nn.Module):
+    def __init__(self, cfg, wav2lip_checkpoint, device='cuda'):
+        super().__init__()
+        self.cfg = cfg
+        self.seq_len = cfg.MODEL.CVAE.SEQ_LEN
+        self.latent_dim = cfg.MODEL.CVAE.LATENT_SIZE
+        self.device = device
+
+        self.audio_encoder = AudioEncoder(wav2lip_checkpoint, device)
+        self.audio_encoder.eval()
+        for param in self.audio_encoder.parameters():
+            param.requires_grad = False
+
+        self.netG = CVAE(cfg)
+        self.netD_motion = PoseSequenceDiscriminator(cfg)
+        
+        
+    def forward(self, x):
+
+        batch = {}
+        coeff_gt = x['gt'].cuda().squeeze(0)           #bs frame_len+1 73
+        batch['pose_motion_gt'] = coeff_gt[:, 1:, 64:70] - coeff_gt[:, :1, 64:70] #bs frame_len 6
+        batch['ref'] = coeff_gt[:, 0, 64:70]  #bs  6
+        batch['class'] = x['class'].squeeze(0).cuda() # bs
+        indiv_mels= x['indiv_mels'].cuda().squeeze(0) # bs seq_len+1 80 16
+
+        # forward
+        audio_emb_list = []
+        audio_emb = self.audio_encoder(indiv_mels[:, 1:, :, :].unsqueeze(2)) #bs seq_len 512
+        batch['audio_emb'] = audio_emb
+        batch = self.netG(batch)
+
+        pose_motion_pred = batch['pose_motion_pred']           # bs frame_len 6
+        pose_gt = coeff_gt[:, 1:, 64:70].clone()               # bs frame_len 6
+        pose_pred = coeff_gt[:, :1, 64:70] + pose_motion_pred  # bs frame_len 6
+
+        batch['pose_pred'] = pose_pred
+        batch['pose_gt'] = pose_gt
+
+        return batch
+
+    def test(self, x):
+
+        batch = {}
+        ref = x['ref']                            #bs 1 70
+        batch['ref'] = x['ref'][:,0,-6:]  
+        batch['class'] = x['class']  
+        bs = ref.shape[0]
+        
+        indiv_mels= x['indiv_mels']               # bs T 1 80 16
+        indiv_mels_use = indiv_mels[:, 1:]        # we regard the ref as the first frame
+        num_frames = x['num_frames']
+        num_frames = int(num_frames) - 1
+
+        #  
+        div = num_frames//self.seq_len
+        re = num_frames%self.seq_len
+        audio_emb_list = []
+        pose_motion_pred_list = [torch.zeros(batch['ref'].unsqueeze(1).shape, dtype=batch['ref'].dtype, 
+                                                device=batch['ref'].device)]
+
+        for i in range(div):
+            z = torch.randn(bs, self.latent_dim).to(ref.device)
+            batch['z'] = z
+            audio_emb = self.audio_encoder(indiv_mels_use[:, i*self.seq_len:(i+1)*self.seq_len,:,:,:]) #bs seq_len 512
+            batch['audio_emb'] = audio_emb
+            batch = self.netG.test(batch)
+            pose_motion_pred_list.append(batch['pose_motion_pred'])  #list of bs seq_len 6
+        
+        if re != 0:
+            z = torch.randn(bs, self.latent_dim).to(ref.device)
+            batch['z'] = z
+            audio_emb = self.audio_encoder(indiv_mels_use[:, -1*self.seq_len:,:,:,:]) #bs seq_len  512
+            if audio_emb.shape[1] != self.seq_len:
+                pad_dim = self.seq_len-audio_emb.shape[1]
+                pad_audio_emb = audio_emb[:, :1].repeat(1, pad_dim, 1) 
+                audio_emb = torch.cat([pad_audio_emb, audio_emb], 1) 
+            batch['audio_emb'] = audio_emb
+            batch = self.netG.test(batch)
+            pose_motion_pred_list.append(batch['pose_motion_pred'][:,-1*re:,:])   
+        
+        pose_motion_pred = torch.cat(pose_motion_pred_list, dim = 1)
+        batch['pose_motion_pred'] = pose_motion_pred
+
+        pose_pred = ref[:, :1, -6:] + pose_motion_pred  # bs T 6
+
+        batch['pose_pred'] = pose_pred
+        return batch
diff --git a/animation/SadTalker/src/audio2pose_models/audio_encoder.py b/animation/SadTalker/src/audio2pose_models/audio_encoder.py
new file mode 100644
index 0000000..6279d20
--- /dev/null
+++ b/animation/SadTalker/src/audio2pose_models/audio_encoder.py
@@ -0,0 +1,64 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+class Conv2d(nn.Module):
+    def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.conv_block = nn.Sequential(
+                            nn.Conv2d(cin, cout, kernel_size, stride, padding),
+                            nn.BatchNorm2d(cout)
+                            )
+        self.act = nn.ReLU()
+        self.residual = residual
+
+    def forward(self, x):
+        out = self.conv_block(x)
+        if self.residual:
+            out += x
+        return self.act(out)
+
+class AudioEncoder(nn.Module):
+    def __init__(self, wav2lip_checkpoint, device):
+        super(AudioEncoder, self).__init__()
+
+        self.audio_encoder = nn.Sequential(
+            Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
+            Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
+            Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True),
+
+            Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1),
+            Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
+            Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),
+
+            Conv2d(64, 128, kernel_size=3, stride=3, padding=1),
+            Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
+            Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),
+
+            Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1),
+            Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True),
+
+            Conv2d(256, 512, kernel_size=3, stride=1, padding=0),
+            Conv2d(512, 512, kernel_size=1, stride=1, padding=0),)
+
+        #### load the pre-trained audio_encoder, we do not need to load wav2lip model here.
+        # wav2lip_state_dict = torch.load(wav2lip_checkpoint, map_location=torch.device(device))['state_dict']
+        # state_dict = self.audio_encoder.state_dict()
+
+        # for k,v in wav2lip_state_dict.items():
+        #     if 'audio_encoder' in k:
+        #         state_dict[k.replace('module.audio_encoder.', '')] = v
+        # self.audio_encoder.load_state_dict(state_dict)
+
+
+    def forward(self, audio_sequences):
+        # audio_sequences = (B, T, 1, 80, 16)
+        B = audio_sequences.size(0)
+
+        audio_sequences = torch.cat([audio_sequences[:, i] for i in range(audio_sequences.size(1))], dim=0)
+
+        audio_embedding = self.audio_encoder(audio_sequences) # B, 512, 1, 1
+        dim = audio_embedding.shape[1]
+        audio_embedding = audio_embedding.reshape((B, -1, dim, 1, 1))
+
+        return audio_embedding.squeeze(-1).squeeze(-1) #B seq_len+1 512 
diff --git a/animation/SadTalker/src/audio2pose_models/cvae.py b/animation/SadTalker/src/audio2pose_models/cvae.py
new file mode 100644
index 0000000..d017ce8
--- /dev/null
+++ b/animation/SadTalker/src/audio2pose_models/cvae.py
@@ -0,0 +1,149 @@
+import torch
+import torch.nn.functional as F
+from torch import nn
+from src.audio2pose_models.res_unet import ResUnet
+
+def class2onehot(idx, class_num):
+
+    assert torch.max(idx).item() < class_num
+    onehot = torch.zeros(idx.size(0), class_num).to(idx.device)
+    onehot.scatter_(1, idx, 1)
+    return onehot
+
+class CVAE(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        encoder_layer_sizes = cfg.MODEL.CVAE.ENCODER_LAYER_SIZES
+        decoder_layer_sizes = cfg.MODEL.CVAE.DECODER_LAYER_SIZES
+        latent_size = cfg.MODEL.CVAE.LATENT_SIZE
+        num_classes = cfg.DATASET.NUM_CLASSES
+        audio_emb_in_size = cfg.MODEL.CVAE.AUDIO_EMB_IN_SIZE
+        audio_emb_out_size = cfg.MODEL.CVAE.AUDIO_EMB_OUT_SIZE
+        seq_len = cfg.MODEL.CVAE.SEQ_LEN
+
+        self.latent_size = latent_size
+
+        self.encoder = ENCODER(encoder_layer_sizes, latent_size, num_classes,
+                                audio_emb_in_size, audio_emb_out_size, seq_len)
+        self.decoder = DECODER(decoder_layer_sizes, latent_size, num_classes,
+                                audio_emb_in_size, audio_emb_out_size, seq_len)
+    def reparameterize(self, mu, logvar):
+        std = torch.exp(0.5 * logvar)
+        eps = torch.randn_like(std)
+        return mu + eps * std
+
+    def forward(self, batch):
+        batch = self.encoder(batch)
+        mu = batch['mu']
+        logvar = batch['logvar']
+        z = self.reparameterize(mu, logvar)
+        batch['z'] = z
+        return self.decoder(batch)
+
+    def test(self, batch):
+        '''
+        class_id = batch['class']
+        z = torch.randn([class_id.size(0), self.latent_size]).to(class_id.device)
+        batch['z'] = z
+        '''
+        return self.decoder(batch)
+
+class ENCODER(nn.Module):
+    def __init__(self, layer_sizes, latent_size, num_classes, 
+                audio_emb_in_size, audio_emb_out_size, seq_len):
+        super().__init__()
+
+        self.resunet = ResUnet()
+        self.num_classes = num_classes
+        self.seq_len = seq_len
+
+        self.MLP = nn.Sequential()
+        layer_sizes[0] += latent_size + seq_len*audio_emb_out_size + 6
+        for i, (in_size, out_size) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])):
+            self.MLP.add_module(
+                name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
+            self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU())
+
+        self.linear_means = nn.Linear(layer_sizes[-1], latent_size)
+        self.linear_logvar = nn.Linear(layer_sizes[-1], latent_size)
+        self.linear_audio = nn.Linear(audio_emb_in_size, audio_emb_out_size)
+
+        self.classbias = nn.Parameter(torch.randn(self.num_classes, latent_size))
+
+    def forward(self, batch):
+        class_id = batch['class']
+        pose_motion_gt = batch['pose_motion_gt']                             #bs seq_len 6
+        ref = batch['ref']                             #bs 6
+        bs = pose_motion_gt.shape[0]
+        audio_in = batch['audio_emb']                          # bs seq_len audio_emb_in_size
+
+        #pose encode
+        pose_emb = self.resunet(pose_motion_gt.unsqueeze(1))          #bs 1 seq_len 6 
+        pose_emb = pose_emb.reshape(bs, -1)                    #bs seq_len*6
+
+        #audio mapping
+        print(audio_in.shape)
+        audio_out = self.linear_audio(audio_in)                # bs seq_len audio_emb_out_size
+        audio_out = audio_out.reshape(bs, -1)
+
+        class_bias = self.classbias[class_id]                  #bs latent_size
+        x_in = torch.cat([ref, pose_emb, audio_out, class_bias], dim=-1) #bs seq_len*(audio_emb_out_size+6)+latent_size
+        x_out = self.MLP(x_in)
+
+        mu = self.linear_means(x_out)
+        logvar = self.linear_means(x_out)                      #bs latent_size 
+
+        batch.update({'mu':mu, 'logvar':logvar})
+        return batch
+
+class DECODER(nn.Module):
+    def __init__(self, layer_sizes, latent_size, num_classes, 
+                audio_emb_in_size, audio_emb_out_size, seq_len):
+        super().__init__()
+
+        self.resunet = ResUnet()
+        self.num_classes = num_classes
+        self.seq_len = seq_len
+
+        self.MLP = nn.Sequential()
+        input_size = latent_size + seq_len*audio_emb_out_size + 6
+        for i, (in_size, out_size) in enumerate(zip([input_size]+layer_sizes[:-1], layer_sizes)):
+            self.MLP.add_module(
+                name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
+            if i+1 < len(layer_sizes):
+                self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU())
+            else:
+                self.MLP.add_module(name="sigmoid", module=nn.Sigmoid())
+        
+        self.pose_linear = nn.Linear(6, 6)
+        self.linear_audio = nn.Linear(audio_emb_in_size, audio_emb_out_size)
+
+        self.classbias = nn.Parameter(torch.randn(self.num_classes, latent_size))
+
+    def forward(self, batch):
+
+        z = batch['z']                                          #bs latent_size
+        bs = z.shape[0]
+        class_id = batch['class']
+        ref = batch['ref']                             #bs 6
+        audio_in = batch['audio_emb']                           # bs seq_len audio_emb_in_size
+        #print('audio_in: ', audio_in[:, :, :10])
+
+        audio_out = self.linear_audio(audio_in)                 # bs seq_len audio_emb_out_size
+        #print('audio_out: ', audio_out[:, :, :10])
+        audio_out = audio_out.reshape([bs, -1])                 # bs seq_len*audio_emb_out_size
+        class_bias = self.classbias[class_id]                   #bs latent_size
+
+        z = z + class_bias
+        x_in = torch.cat([ref, z, audio_out], dim=-1)
+        x_out = self.MLP(x_in)                                  # bs layer_sizes[-1]
+        x_out = x_out.reshape((bs, self.seq_len, -1))
+
+        #print('x_out: ', x_out)
+
+        pose_emb = self.resunet(x_out.unsqueeze(1))             #bs 1 seq_len 6
+
+        pose_motion_pred = self.pose_linear(pose_emb.squeeze(1))       #bs seq_len 6
+
+        batch.update({'pose_motion_pred':pose_motion_pred})
+        return batch
diff --git a/animation/SadTalker/src/audio2pose_models/discriminator.py b/animation/SadTalker/src/audio2pose_models/discriminator.py
new file mode 100644
index 0000000..339c38e
--- /dev/null
+++ b/animation/SadTalker/src/audio2pose_models/discriminator.py
@@ -0,0 +1,76 @@
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+class ConvNormRelu(nn.Module):
+    def __init__(self, conv_type='1d', in_channels=3, out_channels=64, downsample=False,
+                 kernel_size=None, stride=None, padding=None, norm='BN', leaky=False):
+        super().__init__()
+        if kernel_size is None:
+            if downsample:
+                kernel_size, stride, padding = 4, 2, 1
+            else:
+                kernel_size, stride, padding = 3, 1, 1
+
+        if conv_type == '2d':
+            self.conv = nn.Conv2d(
+                in_channels,
+                out_channels,
+                kernel_size,
+                stride,
+                padding,
+                bias=False,
+            )
+            if norm == 'BN':
+                self.norm = nn.BatchNorm2d(out_channels)
+            elif norm == 'IN':
+                self.norm = nn.InstanceNorm2d(out_channels)
+            else:
+                raise NotImplementedError
+        elif conv_type == '1d':
+            self.conv = nn.Conv1d(
+                in_channels,
+                out_channels,
+                kernel_size,
+                stride,
+                padding,
+                bias=False,
+            )
+            if norm == 'BN':
+                self.norm = nn.BatchNorm1d(out_channels)
+            elif norm == 'IN':
+                self.norm = nn.InstanceNorm1d(out_channels)
+            else:
+                raise NotImplementedError
+        nn.init.kaiming_normal_(self.conv.weight)
+
+        self.act = nn.LeakyReLU(negative_slope=0.2, inplace=False) if leaky else nn.ReLU(inplace=True)
+
+    def forward(self, x):
+        x = self.conv(x)
+        if isinstance(self.norm, nn.InstanceNorm1d):
+            x = self.norm(x.permute((0, 2, 1))).permute((0, 2, 1))  # normalize on [C]
+        else:
+            x = self.norm(x)
+        x = self.act(x)
+        return x
+
+
+class PoseSequenceDiscriminator(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        self.cfg = cfg
+        leaky = self.cfg.MODEL.DISCRIMINATOR.LEAKY_RELU
+
+        self.seq = nn.Sequential(
+            ConvNormRelu('1d', cfg.MODEL.DISCRIMINATOR.INPUT_CHANNELS, 256, downsample=True, leaky=leaky),  # B, 256, 64
+            ConvNormRelu('1d', 256, 512, downsample=True, leaky=leaky),  # B, 512, 32
+            ConvNormRelu('1d', 512, 1024, kernel_size=3, stride=1, padding=1, leaky=leaky),  # B, 1024, 16
+            nn.Conv1d(1024, 1, kernel_size=3, stride=1, padding=1, bias=True)  # B, 1, 16
+        )
+
+    def forward(self, x):
+        x = x.reshape(x.size(0), x.size(1), -1).transpose(1, 2)
+        x = self.seq(x)
+        x = x.squeeze(1)
+        return x
\ No newline at end of file
diff --git a/animation/SadTalker/src/audio2pose_models/networks.py b/animation/SadTalker/src/audio2pose_models/networks.py
new file mode 100644
index 0000000..8aa0b13
--- /dev/null
+++ b/animation/SadTalker/src/audio2pose_models/networks.py
@@ -0,0 +1,140 @@
+import torch.nn as nn
+import torch
+
+
+class ResidualConv(nn.Module):
+    def __init__(self, input_dim, output_dim, stride, padding):
+        super(ResidualConv, self).__init__()
+
+        self.conv_block = nn.Sequential(
+            nn.BatchNorm2d(input_dim),
+            nn.ReLU(),
+            nn.Conv2d(
+                input_dim, output_dim, kernel_size=3, stride=stride, padding=padding
+            ),
+            nn.BatchNorm2d(output_dim),
+            nn.ReLU(),
+            nn.Conv2d(output_dim, output_dim, kernel_size=3, padding=1),
+        )
+        self.conv_skip = nn.Sequential(
+            nn.Conv2d(input_dim, output_dim, kernel_size=3, stride=stride, padding=1),
+            nn.BatchNorm2d(output_dim),
+        )
+
+    def forward(self, x):
+
+        return self.conv_block(x) + self.conv_skip(x)
+
+
+class Upsample(nn.Module):
+    def __init__(self, input_dim, output_dim, kernel, stride):
+        super(Upsample, self).__init__()
+
+        self.upsample = nn.ConvTranspose2d(
+            input_dim, output_dim, kernel_size=kernel, stride=stride
+        )
+
+    def forward(self, x):
+        return self.upsample(x)
+
+
+class Squeeze_Excite_Block(nn.Module):
+    def __init__(self, channel, reduction=16):
+        super(Squeeze_Excite_Block, self).__init__()
+        self.avg_pool = nn.AdaptiveAvgPool2d(1)
+        self.fc = nn.Sequential(
+            nn.Linear(channel, channel // reduction, bias=False),
+            nn.ReLU(inplace=True),
+            nn.Linear(channel // reduction, channel, bias=False),
+            nn.Sigmoid(),
+        )
+
+    def forward(self, x):
+        b, c, _, _ = x.size()
+        y = self.avg_pool(x).view(b, c)
+        y = self.fc(y).view(b, c, 1, 1)
+        return x * y.expand_as(x)
+
+
+class ASPP(nn.Module):
+    def __init__(self, in_dims, out_dims, rate=[6, 12, 18]):
+        super(ASPP, self).__init__()
+
+        self.aspp_block1 = nn.Sequential(
+            nn.Conv2d(
+                in_dims, out_dims, 3, stride=1, padding=rate[0], dilation=rate[0]
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(out_dims),
+        )
+        self.aspp_block2 = nn.Sequential(
+            nn.Conv2d(
+                in_dims, out_dims, 3, stride=1, padding=rate[1], dilation=rate[1]
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(out_dims),
+        )
+        self.aspp_block3 = nn.Sequential(
+            nn.Conv2d(
+                in_dims, out_dims, 3, stride=1, padding=rate[2], dilation=rate[2]
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(out_dims),
+        )
+
+        self.output = nn.Conv2d(len(rate) * out_dims, out_dims, 1)
+        self._init_weights()
+
+    def forward(self, x):
+        x1 = self.aspp_block1(x)
+        x2 = self.aspp_block2(x)
+        x3 = self.aspp_block3(x)
+        out = torch.cat([x1, x2, x3], dim=1)
+        return self.output(out)
+
+    def _init_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight)
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+
+
+class Upsample_(nn.Module):
+    def __init__(self, scale=2):
+        super(Upsample_, self).__init__()
+
+        self.upsample = nn.Upsample(mode="bilinear", scale_factor=scale)
+
+    def forward(self, x):
+        return self.upsample(x)
+
+
+class AttentionBlock(nn.Module):
+    def __init__(self, input_encoder, input_decoder, output_dim):
+        super(AttentionBlock, self).__init__()
+
+        self.conv_encoder = nn.Sequential(
+            nn.BatchNorm2d(input_encoder),
+            nn.ReLU(),
+            nn.Conv2d(input_encoder, output_dim, 3, padding=1),
+            nn.MaxPool2d(2, 2),
+        )
+
+        self.conv_decoder = nn.Sequential(
+            nn.BatchNorm2d(input_decoder),
+            nn.ReLU(),
+            nn.Conv2d(input_decoder, output_dim, 3, padding=1),
+        )
+
+        self.conv_attn = nn.Sequential(
+            nn.BatchNorm2d(output_dim),
+            nn.ReLU(),
+            nn.Conv2d(output_dim, 1, 1),
+        )
+
+    def forward(self, x1, x2):
+        out = self.conv_encoder(x1) + self.conv_decoder(x2)
+        out = self.conv_attn(out)
+        return out * x2
\ No newline at end of file
diff --git a/animation/SadTalker/src/audio2pose_models/res_unet.py b/animation/SadTalker/src/audio2pose_models/res_unet.py
new file mode 100644
index 0000000..f2611e1
--- /dev/null
+++ b/animation/SadTalker/src/audio2pose_models/res_unet.py
@@ -0,0 +1,65 @@
+import torch
+import torch.nn as nn
+from src.audio2pose_models.networks import ResidualConv, Upsample
+
+
+class ResUnet(nn.Module):
+    def __init__(self, channel=1, filters=[32, 64, 128, 256]):
+        super(ResUnet, self).__init__()
+
+        self.input_layer = nn.Sequential(
+            nn.Conv2d(channel, filters[0], kernel_size=3, padding=1),
+            nn.BatchNorm2d(filters[0]),
+            nn.ReLU(),
+            nn.Conv2d(filters[0], filters[0], kernel_size=3, padding=1),
+        )
+        self.input_skip = nn.Sequential(
+            nn.Conv2d(channel, filters[0], kernel_size=3, padding=1)
+        )
+
+        self.residual_conv_1 = ResidualConv(filters[0], filters[1], stride=(2,1), padding=1)
+        self.residual_conv_2 = ResidualConv(filters[1], filters[2], stride=(2,1), padding=1)
+
+        self.bridge = ResidualConv(filters[2], filters[3], stride=(2,1), padding=1)
+
+        self.upsample_1 = Upsample(filters[3], filters[3], kernel=(2,1), stride=(2,1))
+        self.up_residual_conv1 = ResidualConv(filters[3] + filters[2], filters[2], stride=1, padding=1)
+
+        self.upsample_2 = Upsample(filters[2], filters[2], kernel=(2,1), stride=(2,1))
+        self.up_residual_conv2 = ResidualConv(filters[2] + filters[1], filters[1], stride=1, padding=1)
+
+        self.upsample_3 = Upsample(filters[1], filters[1], kernel=(2,1), stride=(2,1))
+        self.up_residual_conv3 = ResidualConv(filters[1] + filters[0], filters[0], stride=1, padding=1)
+
+        self.output_layer = nn.Sequential(
+            nn.Conv2d(filters[0], 1, 1, 1),
+            nn.Sigmoid(),
+        )
+
+    def forward(self, x):
+        # Encode
+        x1 = self.input_layer(x) + self.input_skip(x)
+        x2 = self.residual_conv_1(x1)
+        x3 = self.residual_conv_2(x2)
+        # Bridge
+        x4 = self.bridge(x3)
+
+        # Decode
+        x4 = self.upsample_1(x4)
+        x5 = torch.cat([x4, x3], dim=1)
+
+        x6 = self.up_residual_conv1(x5)
+
+        x6 = self.upsample_2(x6)
+        x7 = torch.cat([x6, x2], dim=1)
+
+        x8 = self.up_residual_conv2(x7)
+
+        x8 = self.upsample_3(x8)
+        x9 = torch.cat([x8, x1], dim=1)
+
+        x10 = self.up_residual_conv3(x9)
+
+        output = self.output_layer(x10)
+
+        return output
\ No newline at end of file