From 4eaf68ae58c3795c4aec474b2de9386d307d6a14 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Tue, 12 Nov 2013 02:04:08 -0500 Subject: [PATCH] Lower the buffer size of the Opus decoder based on the data we're actual receiving. Handle invocations of the decoder with null data for packet loss indication. --- jni/nv_opus_dec.c | 4 +- jni/nv_opus_dec_jni.c | 17 ++++-- libs/armeabi-v7a/libnv_opus_dec.so | Bin 148716 -> 152812 bytes .../av/audio/AvAudioDepacketizer.java | 54 +++++++++--------- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/jni/nv_opus_dec.c b/jni/nv_opus_dec.c index 60425aec..c3514a82 100644 --- a/jni/nv_opus_dec.c +++ b/jni/nv_opus_dec.c @@ -30,7 +30,7 @@ int nv_opus_get_channel_count(void) { // This number assumes 2 channels at 48 KHz int nv_opus_get_max_out_shorts(void) { - return 5760*2; + return 512*nv_opus_get_channel_count(); } // The Opus stream is 48 KHz @@ -48,7 +48,7 @@ int nv_opus_decode(unsigned char* indata, int inlen, short* outpcmdata) { // Decoding to 16-bit PCM with FEC off // Maximum length assuming 48KHz sample rate err = opus_decode(decoder, indata, inlen, - outpcmdata, 5760, 0); + outpcmdata, 512, 0); return err; } diff --git a/jni/nv_opus_dec_jni.c b/jni/nv_opus_dec_jni.c index bb99568b..1df69883 100644 --- a/jni/nv_opus_dec_jni.c +++ b/jni/nv_opus_dec_jni.c @@ -1,5 +1,6 @@ #include "nv_opus_dec.h" +#include #include // This function must be called before @@ -48,13 +49,19 @@ Java_com_limelight_nvstream_av_audio_OpusDecoder_decode( jbyte* jni_input_data; jshort* jni_pcm_data; - jni_input_data = (*env)->GetByteArrayElements(env, indata, 0); - jni_pcm_data = (*env)->GetShortArrayElements(env, outpcmdata, 0); + jni_pcm_data = (*env)->GetShortArrayElements(env, outpcmdata, 0); + if (indata != NULL) { + jni_input_data = (*env)->GetByteArrayElements(env, indata, 0); - ret = nv_opus_decode(&jni_input_data[inoff], inlen, jni_pcm_data); + ret = nv_opus_decode(&jni_input_data[inoff], inlen, jni_pcm_data); + + // The input data isn't changed so it can be safely aborted + (*env)->ReleaseByteArrayElements(env, indata, jni_input_data, JNI_ABORT); + } + else { + ret = nv_opus_decode(NULL, 0, jni_pcm_data); + } - // The input data isn't changed so it can be safely aborted - (*env)->ReleaseByteArrayElements(env, indata, jni_input_data, JNI_ABORT); (*env)->ReleaseShortArrayElements(env, outpcmdata, jni_pcm_data, 0); return ret; diff --git a/libs/armeabi-v7a/libnv_opus_dec.so b/libs/armeabi-v7a/libnv_opus_dec.so index 4f8be2974dcd277ee42fb4f04ea4696b1a0b65c9..1c02515a399ab84ba3f12b6485bd54bcf1d4f850 100644 GIT binary patch delta 6731 zcmds+jbBw&y2sbE&N*-(R1io|kyjO85X3i9Qc~`L5hxlhDN!k{@N=mkZN${hQ8Bd- zUbT&Gnq)jCbHld;x4Q9)fn{DTyW!}}Sc7#ynZa-jQ}Z>?{VmQS_V6dn{(PR#cm1B{ zS!=Jg_Fil6ea_YOK38}7Y>K3(H%CpK?3c`$Fbl)s&lmxhGdR0E+hY-C#$sSzGS0z< zEY2F{2;2PO_~94a#JcPbqQX$2(|+5!ctLk-3a4iyt8Jc8U*^rGinKi5nHs?Qvgy!C z%2JB9`=TT=wt@-67_)*=(EGsf2v2uJ{B!8R(6OM-@&Yi?)WX}>a6ZK zVZ3i5E;=mgPk>$;!R)^}3gK8`NT`8cY0$Tz*BJCzOwfAh*O;fl0x@CohyqKXV+{lP z0J?3ICqZ=JF!Y5{@!gKE;kaW+xB+c7#Cw|<+oYW4;dDZ|!ovgVVi-$BbQ#gHD4k9qGAJqWx!~%b+V| zU*0fdR+Hiw8z>#6NSFy-Ahi>^0XjnJdT19k(AnWe6fPU$ZO~?PxIxC<&?lgSqz=R_ zU5gIJNIlYG|5wK}Bs3WkX7Khdup+z-9{1~G9{iw5hU)0HEZ@PMGHjOEH& zE$|j9S1eQP%TpN>3qs_Ecr$j;#@GUxl?WY@#+Xy;nb4IP$d~$M=nK$6Qm=w`WOZJU z&i2ZnL$LL9#dktCJ?z#8yqWTmS3tm~CmBnYZJa~S8RfKB_~^z5#PZ~*3G;c%P}3j! z>TUXUJYmoyl~C{aCl(y&?w<|ba%k+Co&A{$%?y)Y0Nr6|bs2PtvfewM8kHm7;fYSn zYpkrhA2G2Hio>5>cOUMdKZUmDDZbW0c5#X-S?#ZIufr6GE{M&s6}k%X2pRtwTHFs! zQoErmf9#I;@xlECEe>&s!l2JUSIbK_q=z2kgC*REguOB$841fVj;;)MgpH0$Bp7eVeb8;#lFhP%hkNMHppBRIE9i7A z0H-{`w;~^Uo4hsJpp6UWy6B;C#oU7~!HS8M4OsEuGOnN?=)H)S%6L={9S>cH_;MMa z)I+C18<(&>8xG^m`y6xvZnLf_e4&S4C>lV#YnLtRp_f9ZAl|hsSM|{E{;PhLy$44n zDs)}*ozMrNyVjxBLw^QsHmuV>L63y)+AW%(*C}^>cPK@@`f0y^Vq z2jNj77>Q?=U=;R6LA)}Y7sO-#g5XnlEfCCR%q937-XaCD|Fqdf;RQSr1Yg8fFPM)V zUT`7ay9NIfFI|GzLCmhWRd4_V1p`2|fr+1g5T2i`2OOm0S&{bB>hKJZO zOf%cw%GgHNaku*iE3_$NS?YlNkbG-RSjYd?7p5Ja%o&d;OtT(x-@flo;nrgrKRXf% z(*mLIxy?5B?G88XCR>D!cO1APYv)n9FlXOSD) zOl+9VKi?jON*`XEeC&B7x^Mrb-hI0Pc1qfp^>J4^(!y;Q@5DHRZKiw{Wwt$SJ9noo zu{_PocBm#gf0&I$MZn{N$NGMvZKbUt)P4J;TSTVA&ld4$2f9|9W^u*ZhS?I)UES@l zMdwGLrNjWFSZA>4sr`~^U16P4E>0{)s76EMf}7#t?#g{X3Eia#ZB|}S$ogjZcT~1vPQc#R`sD+ zWy9o4bV)0kGMjV1lgj@+F@&;}TTj$eq;_cPJfbs7aOwy;p*@ib@v^exKkieUR$l+!!f!hIu5v`g7ieS->=^))ww7wjTd_axrx zJvxzFz5Dh>hK%ojiHcO1HHQUoRsn*L=>-;27te zb5p$NcUr|aUvo=N7O@9fQByt97-jT#d&sI?{4S8wYUR2soR%p8&Ed328Qr{{GCIrC zl%eq=dC>#B}a!Sz`97PQggZLnkL|asRl)DNPCZ;dRPaR{k)LMkvic zETJUrx%NH1Ehoki8>#u+(}->>$L>edHLdx6IMGavemP1sN;~#&zlo;kuX1{ds9il~ zqOXJXlkRqw=WMnxXEPN#n@c9K_5WO_(dyvQC*1AN_FB`Df|{|+(c+x!rq1u}Q+3)W-s7%408l=5F^wx*1i?=-V7RP6%Q-* zVkY&YT=iBa{YWq9tyy$5lrHN#UZLkXoc#tfo^z}oW(cOS)KnEaWgMIt9!jiZxLNn zS1NSQ_wpJ1UIue~sn0E=;S^l(p}XDUSJGm}9U(5UxI11^E4ENQfAEDqcPn{WX^8&P zA83(`q`ozTd?$X~rLgP7atke%5rIRN5QD0F{Kl+A=ubEBnGbS#`Lpx_KcOPsQ9qw0m z5_9V7zo74Z=%l*vJf`Fi`kM2!k!YeG{1=*L4js1H-JXYAF4Cjbk||>eqr*eQF#R zJlf8J274SyVD)~hqaoIl*a&8#fuNBdEx5=3fTLRK6QF1~HqH}I1!Iw)3+^6#z!5ac z9g_KPW_`j6h_l78b+B!)y|4z@ z1z4L($vk?xnd6lRQ-NPtI=@uK7i$81AtugJVGjKAB;tkmaI1t}fcfE*CdML~#m_c= zoGtH#*Q`G3Ysq|+Wp5wO3e+9RJl;|eC5nF*#mpE{^m&}e+mi>Q7|7YxaI}h}HkPw$ z*qJyq1Zx_J#z%1$1iJ~H2WG<(U=hq0{sm-B;Ota9XC?5(f-bNeR?wN_&t4v~hjPbo zHV{@Y8cgKuJXkuK_hIQMh(nC3YMy%XalSDC1H$Wh=Oo}^!RqFyFHhz(?0KHdMuu_Y z%Rgxt<_0Ey1{a6;IK4BBvxCFYHCQRE5SA{=@Usz!quhYumB5N&O|Uwc_>f%(R>B%E z;gX8d(m5=d=4_GH0<*;pVR)Oo1I7@j9*AwMD_!c847c2*h!2(zzVr)F- z5w;hWIsqq#wM~S766O=Eg;`NPkNiy{-*C7FFOK2?|jVF_x{G#;qd zP33+*zy5uqo<)j9OjM(KYbuYi;0K4x8S0>EJlcv*rDWkmo_e{{crOcF=~?P)GFpeI zS*=M$t9xa%Axr&4MDbx&`y`UpNO8^wA}TIQw%R)tQ54PDqHbr8GE;c~Wvg>j`B32< Q=<#mzct4ij3!Ufre*>4_oDjIB1Ko~_O6Os^h9o!)aK4!$xq+287 zW64NTn8HL?jAVn4QLxm)5u+x?Pfc7kVwSkXAj$@XBp6YN8hihjzJv<>eTU!upL6cB zZr!eGT6bGoH(H8=Xv*8cQzz9=kQg@;!^e{`0&YRyN^zOh$XF=MO~fVe=Wplvo+s2_7@Q+Zm3)S2qV`mq_%q4G)!vGeYD z#=GDktE+pi_d^Fkhl5-n12;U&ScA|P!RglS_;u)QY2EQV&`S{K1LXCsnD1Y7`Z?&8 zR%ZV-;XOEB*CkwsUZc|iSdc>K9Oi0p4qqVg2?k$*9vs1#8RcMfxDMJ!r{9OJ9qFpT z2XqAb^vLLL$3-}9=@RZix9fBt17oY@b5bBx%D1IJuZmE{5>cHumW7y)>~DxhY`P)P z9*K@q;p4HhV7;!L2Iwzz`T=ySP7g3L7N;9mG<2~}CqWlPx+coop9}4PE*E22YQ(CB zMqxFCqX-EJ(0M|aLGOdM3Vj0l3uvHoz?V?Cq>DF0w?Us2aVK;o2J9oWH@0*+1{f-I zq{;qU$FoQ{t4ql2p%*~6>f-hudMmVNOm_#BJ@jYLV|4Mac>{V!3v}Wbtb!QfQ|NWj zShda>`{FLBgpLz>EOaw8-%EUnQ=!{*dI9vIAFx41JRdsrF|ks(c>bK%;i&H*Rw5jm zpc|lzg;t;o9uc#O26jOo`iXuW-~s5|d&Kgv2YG`v&|~DIra&)#D`$&Zjqqm6w@s7n zxrvC2Tt9cl%501+6j`I7jY;qcoe5o>f_$MDL7#^95qd3jUK(Rq(9XU~pp7_sy5hT_ z&pz$e2i=+ck(-y->Y0qii#k`3Q!Sr!3yiIKLM%tjny{XSbTvJp>mJgt^9g|Vll|SJ zr(_)L9-j@~ba7 z-E|)6p+A8(XUM(GL+reZ3Q_HU;9ia;;6vbNOA~YnuD?~pA3*c{P%pF-y7Y>Ms`q zu=mhK&}Qtn91$x z(G0D>Veaw~>hG9G(D}Gy!bJn-UcbGAe4tAaFB0+K9y%Jj0`b)%KB0$Bg4W-{_6#`m zoA){B!PsV9OZajRy_h$Ec-JYrq=#M!9f^3?ZMn9G-t=2Nm%Rf=F)DOj^Igzo&|UYT z+CzT|-KL9w3GIhX+jUyhLob(q?zK~``duqgth&DU4KwwT7xzyhPr0gph%XXY0c;Db z1hxj&PrluMJXzGB0jtR53x|ii(@!QZd8~gr`N)O-ir4Sp72i)RA7yVjYTz*HE6*7? znR?4x28P=4h%*3zX2Dj$Ho`%em@1gzqMc<5N0wl=V2MqW7&G@9?X~DCCUkF|jtP`vkY!JM~iT+u$ zaI^}x3APJ%2tE;H>8=471`%a zm@Jqom?4-cm?fAkm?O9_U9xv>o?PL`6U-M}E?6MATF@c5PH=-@kzlc4iD0SVHo-E% z@^rgvDQmE_zBoN-_@Krhif3e)ZmeyF?Meq*CWWj;yc+pd+Z5Yy_{?8VCy)bMZNp)fB z(1>@9^n!Xk>RES7(IaPzIU(xyCe<=lu~3w}bJ9&}RtqOTFG+2m$&FKn(iqt?^#l!7 z&rN-a=&~GFV0e^ zRHf(MBpR<4{_%67)9N3x6{3GvZ@=K(i<;D5{xrgk^3;e|cNwX_d|~N$GOMO#BP4oX zjr~g!(Fl3vUw28;9eK!#q2w=5Tk);$joZ!^qi26ST9)0QfomG~SqC-MtG_R}VhU?t zgI5a}gFQAfb`EchxXn(z*6+<{@a}gKuX3kK)z~+7`BI5`RenjL6m{&o10?!PO)E|D z!05J?_jU76m`|+zWi%tI`pqHMk%>nl_0*nY68(qz`aee)XurI>@*XXbw^!ZsE!c`D zwwZX!6T5PVGy29h^}~bFW*Vzr{nsF(diCzHo4wq!3yCEcs@Kk(G1C)u=fzb-+tq-} zvnBMsTBTNfeNi&Srx9yV3+qo1#mUpI@1@@A-Rs^GZIDej0?8qV;ctOF?Zytu z=qxA8rp6FUd=jxZ8?gd8wsA9-{8Zx#qopJPGb#0-WL2kpcT=KNIqc4B_`BuKOB5sD zzq5h{tFN@|?PIDKi|2MV;E_u7M85nugzl;L9|xj;)%TZUM5EQqPY)O*MjA~=mD@%dME|9kO!U5ib}Bo~G>jDOoSD`e=(P3+4?^q7im4C1 z={xSUvt@4Y!o~z^+0Ag~Xms~1yAi8x??Z~@AD&1o5ib$RL3o(`!Rg%Rk3$i5Qc3fo z$zh8=b+))6-H572Y=imG9x>aSxHfneHn_w0f>Py$k!5M0d6Bz;KGZJw&_@RU`e(4P zcqc#k0$${Nh+RpW@9DsAHpR+Yewf!g$~S(PNTK58kBPjYMETPIDp02RQwvR0Vh7SZ z8m4U=NDGYANBe#V$-QW?wkCp7i25rRhSPw4p2wXnKY#mHW83EA%}#uwj|tNr45wL= zTWbih+7NAK6wM$hBUpKFBt0W#S+%c5(m0|7t#=G1k<|XPGB=j~NCjF|EIm&Yr1c(6 zD-1M7DI7!a+{e$*(3MOl+YsI&eayJ!&PEagq^dl|O zeomuf{&ZG5{3i+wrUlB?jdUPj`iIVz;rll=`bCs9ZDQ*hOjymeR;(y4M3A;~6S+&Y zL)l$K(f%KlJ6n8^mc#zuC3wMxKSbkD8y6kdc+Y5o5}l2Ix; zr44+C{zi09*&@@GUL7@ft%H>u)0URdaPnR9fwRTrv96JK$1fz`19?i-7CIq$BIjLl zGt&rd<$uyL6aBaL{VsamKo7O)AJGJI)3!TXJbzJM{EkK_hbw3=w&csZ$xh^@+}MM) zuxRk%=+Lz3`)IvID&4Mqd6+&nNGG>y?;NFZl60#?JAaJM7^q!)`vm>iE8x~LJos&7 zj9;&(cnD~4_QlQL=6q!rv31(^^YpESK386^#g^EmZLOuvM3c3MujpALjZ|K_hSd&N z3a?=?BDJb(G?}EV<;qY0OP@$Z%akMa^b@_L?EZ$9(NrzqI(eIEq89sq^t_u?vsC#< zE4@K~*Lr_XZ}y{p%G*vFpd4~is6Em8)LvOx>Bs>q3{?&XxZAzTQ3mp(sZ_+vJGtNS zMEE^|Q15`zQ5VeXS2*HBp4y9@PJ41F>YJ+^r68|g3s&^4a)5biQ`+_b4VwtQqEGkeCBMqVRY= z-a^V@bubS+pSRTQ?T33~_v3NX&;L7nTQS0hkphG0)%QdZHWy z-*V(cf~8=Jkjwt5^c=6G}tt9}N47=vwq)xpY zv!p;v308nxsIp|X5X&Cnoc6qnCy!(Z>uCt&22iWrkMgRZ+ diff --git a/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java b/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java index beffca91..13a78f4b 100644 --- a/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java +++ b/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java @@ -11,7 +11,7 @@ public class AvAudioDepacketizer { private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(15); - private AvShortBufferPool pool = new AvShortBufferPool(512); + private AvShortBufferPool pool = new AvShortBufferPool(OpusDecoder.getMaxOutputShorts()); // Sequencing state private short lastSequenceNumber; @@ -21,34 +21,11 @@ public class AvAudioDepacketizer { pool.purge(); } - public void decodeInputData(AvRtpPacket packet) + private void decodeData(byte[] data, int off, int len) { - short seq = packet.getSequenceNumber(); - - if (packet.getPacketType() != 97) { - // Only type 97 is audio - return; - } - - // Toss out the current NAL if we receive a packet that is - // out of sequence - if (lastSequenceNumber != 0 && - (short)(lastSequenceNumber + 1) != seq) - { - System.out.println("Received OOS audio data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); - - // Tell the decoder about this packet loss - //OpusDecoder.decode(null, 0, 0, null); - } - - lastSequenceNumber = seq; - - // This is all the depacketizing we need to do - AvByteBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); - // Submit this data to the decoder short[] pcmData = pool.allocate(); - int decodeLen = OpusDecoder.decode(rtpPayload.data, rtpPayload.offset, rtpPayload.length, pcmData); + int decodeLen = OpusDecoder.decode(data, off, len, pcmData); if (decodeLen > 0) { // Return value of decode is frames decoded per channel @@ -66,6 +43,31 @@ public class AvAudioDepacketizer { } } + public void decodeInputData(AvRtpPacket packet) + { + short seq = packet.getSequenceNumber(); + + if (packet.getPacketType() != 97) { + // Only type 97 is audio + return; + } + + // Toss out the current NAL if we receive a packet that is + // out of sequence + if (lastSequenceNumber != 0 && + (short)(lastSequenceNumber + 1) != seq) + { + System.out.println("Received OOS audio data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); + decodeData(null, 0, 0); + } + + lastSequenceNumber = seq; + + // This is all the depacketizing we need to do + AvByteBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); + decodeData(rtpPayload.data, rtpPayload.offset, rtpPayload.length); + } + public void releaseBuffer(AvShortBufferDescriptor decodedData) { pool.free(decodedData.data);