Bump version to 3.1.12

app/build.gradle

@@ -5,14 +5,14 @@ apply plugin: 'com.android.application'
 
 android {
     compileSdkVersion 23
-    buildToolsVersion "23.0.1"
+    buildToolsVersion "23.0.2"
 
     defaultConfig {
         minSdkVersion 16
         targetSdkVersion 23
 
-        versionName "3.1.11"
+        versionName "3.1.12"
-        versionCode = 66
+        versionCode = 69
     }
 
     productFlavors {

app/src/main/java/com/limelight/Game.java

@@ -1,7 +1,6 @@
 package com.limelight;
 
 
-import com.limelight.LimelightBuildProps;
 import com.limelight.binding.PlatformBinding;
 import com.limelight.binding.input.ControllerHandler;
 import com.limelight.binding.input.KeyboardTranslator;
@@ -64,6 +63,9 @@ public class Game extends Activity implements SurfaceHolder.Callback,
     private final TouchContext[] touchContextMap = new TouchContext[2];
     private long threeFingerDownTime = 0;
 
+    private static final double REFERENCE_HORIZ_RES = 1280;
+    private static final double REFERENCE_VERT_RES = 720;
+
     private static final int THREE_FINGER_TAP_THRESHOLD = 300;
 
     private ControllerHandler controllerHandler;
@@ -77,6 +79,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
     private boolean displayedFailureDialog = false;
     private boolean connecting = false;
     private boolean connected = false;
+    private boolean deferredSurfaceResize = false;
 
     private EvdevWatcher evdevWatcher;
     private int modifierFlags = 0;
@@ -207,17 +210,36 @@ public class Game extends Activity implements SurfaceHolder.Callback,
         InputManager inputManager = (InputManager) getSystemService(Context.INPUT_SERVICE);
         inputManager.registerInputDeviceListener(controllerHandler, null);
 
+        boolean aspectRatioMatch = false;
+        if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.KITKAT) {
+            // On KitKat and later (where we can use the whole screen via immersive mode), we'll
+            // calculate whether we need to scale by aspect ratio or not. If not, we'll use
+            // setFixedSize so we can handle 4K properly. The only known devices that have
+            // >= 4K screens have exactly 4K screens, so we'll be able to hit this good path
+            // on these devices. On Marshmallow, we can start changing to 4K manually but no
+            // 4K devices run 6.0 at the moment.
+            double screenAspectRatio = ((double)screenSize.y) / screenSize.x;
+            double streamAspectRatio = ((double)prefConfig.height) / prefConfig.width;
+            if (Math.abs(screenAspectRatio - streamAspectRatio) < 0.001) {
+                LimeLog.info("Stream has compatible aspect ratio with output display");
+                aspectRatioMatch = true;
+            }
+        }
+
         SurfaceHolder sh = sv.getHolder();
-        if (prefConfig.stretchVideo || !decoderRenderer.isHardwareAccelerated()) {
+        if (prefConfig.stretchVideo || !decoderRenderer.isHardwareAccelerated() || aspectRatioMatch) {
             // Set the surface to the size of the video
             sh.setFixedSize(prefConfig.width, prefConfig.height);
         }
+        else {
+            deferredSurfaceResize = true;
+        }
 
         // Initialize touch contexts
         for (int i = 0; i < touchContextMap.length; i++) {
             touchContextMap[i] = new TouchContext(conn, i,
-                    ((double)prefConfig.width / (double)screenSize.x),
+                    (REFERENCE_HORIZ_RES / (double)screenSize.x),
-                    ((double)prefConfig.height / (double)screenSize.y));
+                    (REFERENCE_VERT_RES / (double)screenSize.y));
         }
 
         if (LimelightBuildProps.ROOT_BUILD) {
@@ -681,8 +703,8 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
             // Scale the deltas if the device resolution is different
             // than the stream resolution
-            deltaX = (int)Math.round((double)deltaX * ((double)prefConfig.width / (double)screenSize.x));
+            deltaX = (int)Math.round((double)deltaX * (REFERENCE_HORIZ_RES / (double)screenSize.x));
-            deltaY = (int)Math.round((double)deltaY * ((double)prefConfig.height / (double)screenSize.y));
+            deltaY = (int)Math.round((double)deltaY * (REFERENCE_VERT_RES / (double)screenSize.y));
 
             conn.sendMouseMove((short)deltaX, (short)deltaY);
         }
@@ -800,7 +822,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
             // Resize the surface to match the aspect ratio of the video
             // This must be done after the surface is created.
-            if (!prefConfig.stretchVideo && decoderRenderer.isHardwareAccelerated()) {
+            if (deferredSurfaceResize) {
                 resizeSurfaceWithAspectRatio((SurfaceView) findViewById(R.id.surfaceView),
                         prefConfig.width, prefConfig.height);
             }

app/src/main/java/com/limelight/binding/audio/AndroidAudioRenderer.java

@@ -9,14 +9,13 @@ import com.limelight.nvstream.av.audio.AudioRenderer;
 
 public class AndroidAudioRenderer implements AudioRenderer {
 
-    private static final int FRAME_SIZE = 960;
-
     private AudioTrack track;
 
     @Override
-    public boolean streamInitialized(int channelCount, int sampleRate) {
+    public boolean streamInitialized(int channelCount, int channelMask, int samplesPerFrame, int sampleRate) {
         int channelConfig;
         int bufferSize;
+        int bytesPerFrame = (samplesPerFrame * 2);
 
         switch (channelCount)
         {
@@ -26,6 +25,12 @@ public class AndroidAudioRenderer implements AudioRenderer {
         case 2:
             channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
             break;
+        case 4:
+            channelConfig = AudioFormat.CHANNEL_OUT_QUAD;
+            break;
+        case 6:
+            channelConfig = AudioFormat.CHANNEL_OUT_5POINT1;
+            break;
         default:
             LimeLog.severe("Decoder returned unhandled channel count");
             return false;
@@ -38,7 +43,7 @@ public class AndroidAudioRenderer implements AudioRenderer {
         // use the recommended larger buffer size.
         try {
             // Buffer two frames of audio if possible
-            bufferSize = FRAME_SIZE * 2;
+            bufferSize = bytesPerFrame * 2;
 
             track = new AudioTrack(AudioManager.STREAM_MUSIC,
                     sampleRate,
@@ -59,10 +64,10 @@ public class AndroidAudioRenderer implements AudioRenderer {
             bufferSize = Math.max(AudioTrack.getMinBufferSize(sampleRate,
                             channelConfig,
                             AudioFormat.ENCODING_PCM_16BIT),
-                    FRAME_SIZE * 2);
+                    bytesPerFrame * 2);
 
             // Round to next frame
-            bufferSize = (((bufferSize + (FRAME_SIZE - 1)) / FRAME_SIZE) * FRAME_SIZE);
+            bufferSize = (((bufferSize + (bytesPerFrame - 1)) / bytesPerFrame) * bytesPerFrame);
 
             track = new AudioTrack(AudioManager.STREAM_MUSIC,
                     sampleRate,

app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java

@@ -4,6 +4,7 @@ import java.nio.ByteBuffer;
 import java.util.Locale;
 import java.util.concurrent.locks.LockSupport;
 
+import org.jcodec.codecs.h264.H264Utils;
 import org.jcodec.codecs.h264.io.model.SeqParameterSet;
 import org.jcodec.codecs.h264.io.model.VUIParameters;
 
@@ -490,7 +491,9 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
                 // Skip to the start of the NALU data
                 spsBuf.position(header.offset+5);
 
-                SeqParameterSet sps = SeqParameterSet.read(spsBuf);
+                // The H264Utils.readSPS function safely handles
+                // Annex B NALUs (including NALUs with escape sequences)
+                SeqParameterSet sps = H264Utils.readSPS(spsBuf);
 
                 // Some decoders rely on H264 level to decide how many buffers are needed
                 // Since we only need one frame buffered, we'll set the level as low as we can
@@ -571,8 +574,10 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
                 // Write the annex B header
                 buf.put(header.data, header.offset, 5);
 
-                // Write the modified SPS to the input buffer
+                // The H264Utils.writeSPS function safely handles
-                sps.write(buf);
+                // Annex B NALUs (including NALUs with escape sequences)
+                ByteBuffer escapedNalu = H264Utils.writeSPS(sps, header.length);
+                buf.put(escapedNalu);
 
                 queueInputBuffer(inputBufferIndex,
                         0, buf.position(),

app/src/main/java/com/limelight/computers/ComputerManagerService.java

@@ -31,11 +31,12 @@ import android.os.IBinder;
 import org.xmlpull.v1.XmlPullParserException;
 
 public class ComputerManagerService extends Service {
-    private static final int SERVERINFO_POLLING_PERIOD_MS = 3000;
+    private static final int SERVERINFO_POLLING_PERIOD_MS = 1500;
     private static final int APPLIST_POLLING_PERIOD_MS = 30000;
+    private static final int APPLIST_FAILED_POLLING_RETRY_MS = 2000;
     private static final int MDNS_QUERY_PERIOD_MS = 1000;
     private static final int FAST_POLL_TIMEOUT = 500;
-    private static final int OFFLINE_POLL_TRIES = 3;
+    private static final int OFFLINE_POLL_TRIES = 5;
 
     private final ComputerManagerBinder binder = new ComputerManagerBinder();
 
@@ -119,7 +120,7 @@ public class ComputerManagerService extends Service {
         return true;
     }
 
-    private Thread createPollingThread(final ComputerDetails details) {
+    private Thread createPollingThread(final PollingTuple tuple) {
         Thread t = new Thread() {
             @Override
             public void run() {
@@ -127,24 +128,26 @@ public class ComputerManagerService extends Service {
                 int offlineCount = 0;
                 while (!isInterrupted() && pollingActive) {
                     try {
+                        // Only allow one request to the machine at a time
+                        synchronized (tuple.networkLock) {
                             // Check if this poll has modified the details
-                        if (!runPoll(details, false, offlineCount)) {
+                            if (!runPoll(tuple.computer, false, offlineCount)) {
-                            LimeLog.warning(details.name + " is offline (try " + offlineCount + ")");
+                                LimeLog.warning(tuple.computer.name + " is offline (try " + offlineCount + ")");
                                 offlineCount++;
-                        }
+                            } else {
-                        else {
                                 offlineCount = 0;
                             }
+                        }
 
                         // Wait until the next polling interval
-                        Thread.sleep(SERVERINFO_POLLING_PERIOD_MS / ((offlineCount > 0) ? 2 : 1));
+                        Thread.sleep(SERVERINFO_POLLING_PERIOD_MS);
                     } catch (InterruptedException e) {
                         break;
                     }
                 }
             }
         };
-        t.setName("Polling thread for "+details.localIp.getHostAddress());
+        t.setName("Polling thread for " + tuple.computer.localIp.getHostAddress());
         return t;
     }
 
@@ -166,7 +169,7 @@ public class ComputerManagerService extends Service {
                         // Report this computer initially
                         listener.notifyComputerUpdated(tuple.computer);
 
-                        tuple.thread = createPollingThread(tuple.computer);
+                        tuple.thread = createPollingThread(tuple);
                         tuple.thread.start();
                     }
                 }
@@ -283,7 +286,7 @@ public class ComputerManagerService extends Service {
 
                     // Start a polling thread if polling is active
                     if (pollingActive && tuple.thread == null) {
-                        tuple.thread = createPollingThread(details);
+                        tuple.thread = createPollingThread(tuple);
                         tuple.thread.start();
                     }
 
@@ -293,7 +296,10 @@ public class ComputerManagerService extends Service {
             }
 
             // If we got here, we didn't find an entry
-            PollingTuple tuple = new PollingTuple(details, pollingActive ? createPollingThread(details) : null);
+            PollingTuple tuple = new PollingTuple(details, null);
+            if (pollingActive) {
+                tuple.thread = createPollingThread(tuple);
+            }
             pollingTuples.add(tuple);
             if (tuple.thread != null) {
                 tuple.thread.start();
@@ -607,6 +613,7 @@ public class ComputerManagerService extends Service {
         private Thread thread;
         private final ComputerDetails computer;
         private final Object pollEvent = new Object();
+        private boolean receivedAppList = false;
 
         public ApplistPoller(ComputerDetails computer) {
             this.computer = computer;
@@ -621,8 +628,16 @@ public class ComputerManagerService extends Service {
         private boolean waitPollingDelay() {
             try {
                 synchronized (pollEvent) {
+                    if (receivedAppList) {
+                        // If we've already reported an app list successfully,
+                        // wait the full polling period
                         pollEvent.wait(APPLIST_POLLING_PERIOD_MS);
                     }
+                    else {
+                        // If we've failed to get an app list so far, retry much earlier
+                        pollEvent.wait(APPLIST_FAILED_POLLING_RETRY_MS);
+                    }
+                }
             } catch (InterruptedException e) {
                 return false;
             }
@@ -630,6 +645,18 @@ public class ComputerManagerService extends Service {
             return thread != null && !thread.isInterrupted();
         }
 
+        private PollingTuple getPollingTuple(ComputerDetails details) {
+            synchronized (pollingTuples) {
+                for (PollingTuple tuple : pollingTuples) {
+                    if (details.uuid.equals(tuple.computer.uuid)) {
+                        return tuple;
+                    }
+                }
+            }
+
+            return null;
+        }
+
         public void start() {
             thread = new Thread() {
                 @Override
@@ -660,9 +687,23 @@ public class ComputerManagerService extends Service {
                         NvHTTP http = new NvHTTP(selectedAddr, idManager.getUniqueId(),
                                 null, PlatformBinding.getCryptoProvider(ComputerManagerService.this));
 
+                        PollingTuple tuple = getPollingTuple(computer);
+
                         try {
-                            // Query the app list from the server
+                            String appList;
-                            String appList = http.getAppListRaw();
+                            if (tuple != null) {
+                                // If we're polling this machine too, grab the network lock
+                                // while doing the app list request to prevent other requests
+                                // from being issued in the meantime.
+                                synchronized (tuple.networkLock) {
+                                    appList = http.getAppListRaw();
+                                }
+                            }
+                            else {
+                                // No polling is happening now, so we just call it directly
+                                appList = http.getAppListRaw();
+                            }
+
                             List<NvApp> list = NvHTTP.getAppListByReader(new StringReader(appList));
                             if (appList != null && !appList.isEmpty() && !list.isEmpty()) {
                                 // Open the cache file
@@ -682,6 +723,7 @@ public class ComputerManagerService extends Service {
 
                                 // Update the computer
                                 computer.rawAppList = appList;
+                                receivedAppList = true;
 
                                 // Notify that the app list has been updated
                                 // and ensure that the thread is still active
@@ -718,10 +760,12 @@ public class ComputerManagerService extends Service {
 class PollingTuple {
     public Thread thread;
     public final ComputerDetails computer;
+    public final Object networkLock;
 
     public PollingTuple(ComputerDetails computer, Thread thread) {
         this.computer = computer;
         this.thread = thread;
+        this.networkLock = new Object();
     }
 }
 

app/src/main/jni/nv_opus_dec/nv_opus_dec.c

@@ -1,16 +1,20 @@
 #include <stdlib.h>
-#include <opus.h>
+#include <opus_multistream.h>
 #include "nv_opus_dec.h"
 
-OpusDecoder* decoder;
+OpusMSDecoder* decoder;
 
 // This function must be called before
 // any other decoding functions
-int nv_opus_init(void) {
+int nv_opus_init(int sampleRate, int channelCount, int streams,
+				 int coupledStreams, const unsigned char *mapping) {
 	int err;
-	decoder = opus_decoder_create(
+	decoder = opus_multistream_decoder_create(
-		nv_opus_get_sample_rate(),
+			sampleRate,
-		nv_opus_get_channel_count(),
+			channelCount,
+			streams,
+			coupledStreams,
+			mapping,
 			&err);
 	return err;
 }
@@ -19,36 +23,20 @@ int nv_opus_init(void) {
 // decoding is finished
 void nv_opus_destroy(void) {
 	if (decoder != NULL) {
-		opus_decoder_destroy(decoder);
+		opus_multistream_decoder_destroy(decoder);
 	}
 }
 
-// The Opus stream is stereo
-int nv_opus_get_channel_count(void) {
-	return 2;
-}
-
-// This number assumes 16-bit samples at 48 KHz with 2.5 ms frames
-int nv_opus_get_max_out_shorts(void) {
-	return 240*nv_opus_get_channel_count();
-}
-
-// The Opus stream is 48 KHz
-int nv_opus_get_sample_rate(void) {
-	return 48000;
-}
-
-// outpcmdata must be 5760*2 shorts in length
 // packets must be decoded in order
 // a packet loss must call this function with NULL indata and 0 inlen
 // returns the number of decoded samples
-int nv_opus_decode(unsigned char* indata, int inlen, short* outpcmdata) {
+int nv_opus_decode(unsigned char* indata, int inlen, short* outpcmdata, int framesize) {
 	int err;
 
 	// Decoding to 16-bit PCM with FEC off
 	// Maximum length assuming 48KHz sample rate
-	err = opus_decode(decoder, indata, inlen,
+	err = opus_multistream_decode(decoder, indata, inlen,
-		outpcmdata, 512, 0);
+		outpcmdata, framesize, 0);
 
 	return err;
 }

app/src/main/jni/nv_opus_dec/nv_opus_dec.h

@@ -1,6 +1,4 @@
-int nv_opus_init(void);
+int nv_opus_init(int sampleRate, int channelCount, int streams,
+                 int coupledStreams, const unsigned char *mapping);
 void nv_opus_destroy(void);
-int nv_opus_get_channel_count(void);
+int nv_opus_decode(unsigned char* indata, int inlen, short* outpcmdata, int framesize);
-int nv_opus_get_max_out_shorts(void);
-int nv_opus_get_sample_rate(void);
-int nv_opus_decode(unsigned char* indata, int inlen, short* outpcmdata);

app/src/main/jni/nv_opus_dec/nv_opus_dec_jni.c

@@ -3,11 +3,26 @@
 #include <stdlib.h>
 #include <jni.h>
 
+static int SamplesPerChannel;
+static int ChannelCount;
+
 // This function must be called before
 // any other decoding functions
 JNIEXPORT jint JNICALL
-Java_com_limelight_nvstream_av_audio_OpusDecoder_init(JNIEnv *env, jobject this) {
+Java_com_limelight_nvstream_av_audio_OpusDecoder_init(JNIEnv *env, jobject this, int sampleRate,
-	return nv_opus_init();
+													  int samplesPerChannel, int channelCount, int streams,
+													  int coupledStreams, jbyteArray mapping) {
+	jbyte* jni_mapping_data;
+	jint ret;
+
+	SamplesPerChannel = samplesPerChannel;
+	ChannelCount = channelCount;
+
+	jni_mapping_data = (*env)->GetByteArrayElements(env, mapping, 0);
+	ret =  nv_opus_init(sampleRate, channelCount, streams, coupledStreams, jni_mapping_data);
+	(*env)->ReleaseByteArrayElements(env, mapping, jni_mapping_data, JNI_ABORT);
+
+	return ret;
 }
 
 // This function must be called after
@@ -17,28 +32,9 @@ Java_com_limelight_nvstream_av_audio_OpusDecoder_destroy(JNIEnv *env, jobject th
 	nv_opus_destroy();
 }
 
-// The Opus stream is stereo
-JNIEXPORT jint JNICALL
-Java_com_limelight_nvstream_av_audio_OpusDecoder_getChannelCount(JNIEnv *env, jobject this) {
-	return nv_opus_get_channel_count();
-}
-
-// This number assumes 2 channels at 48 KHz
-JNIEXPORT jint JNICALL
-Java_com_limelight_nvstream_av_audio_OpusDecoder_getMaxOutputShorts(JNIEnv *env, jobject this) {
-	return nv_opus_get_max_out_shorts();
-}
-
-// The Opus stream is 48 KHz
-JNIEXPORT jint JNICALL
-Java_com_limelight_nvstream_av_audio_OpusDecoder_getSampleRate(JNIEnv *env, jobject this) {
-	return nv_opus_get_sample_rate();
-}
-
-// outpcmdata must be 5760*2 shorts in length
 // packets must be decoded in order
 // a packet loss must call this function with NULL indata and 0 inlen
-// returns the number of decoded samples
+// returns the number of decoded bytes
 JNIEXPORT jint JNICALL
 Java_com_limelight_nvstream_av_audio_OpusDecoder_decode(
 	JNIEnv *env, jobject this, // JNI parameters
@@ -53,13 +49,18 @@ Java_com_limelight_nvstream_av_audio_OpusDecoder_decode(
 	if (indata != NULL) {
 		jni_input_data = (*env)->GetByteArrayElements(env, indata, 0);
 
-		ret = nv_opus_decode(&jni_input_data[inoff], inlen, (jshort*)jni_pcm_data);
+		ret = nv_opus_decode(&jni_input_data[inoff], inlen, (jshort*)jni_pcm_data, SamplesPerChannel);
 
 		// The input data isn't changed so it can be safely aborted
 		(*env)->ReleaseByteArrayElements(env, indata, jni_input_data, JNI_ABORT);
 	}
 	else {
-		ret = nv_opus_decode(NULL, 0, (jshort*)jni_pcm_data);
+		ret = nv_opus_decode(NULL, 0, (jshort*)jni_pcm_data, SamplesPerChannel);
+	}
+
+	// Convert samples (2 bytes) per channel to total bytes returned
+	if (ret > 0) {
+		ret *= ChannelCount * 2;
 	}
 
 	(*env)->ReleaseByteArrayElements(env, outpcmdata, jni_pcm_data, 0);