Adapt to new-core reworking of moonlight-common (likely buggy)

app/src/main/java/com/limelight/Game.java

@@ -10,16 +10,15 @@ import com.limelight.binding.input.TouchContext;
 import com.limelight.binding.input.driver.UsbDriverService;
 import com.limelight.binding.input.evdev.EvdevListener;
 import com.limelight.binding.input.virtual_controller.VirtualController;
-import com.limelight.binding.video.EnhancedDecoderRenderer;
 import com.limelight.binding.video.MediaCodecDecoderRenderer;
 import com.limelight.binding.video.MediaCodecHelper;
 import com.limelight.nvstream.NvConnection;
 import com.limelight.nvstream.NvConnectionListener;
 import com.limelight.nvstream.StreamConfiguration;
-import com.limelight.nvstream.av.video.VideoDecoderRenderer;
 import com.limelight.nvstream.http.NvApp;
 import com.limelight.nvstream.input.KeyboardPacket;
 import com.limelight.nvstream.input.MouseButtonPacket;
+import com.limelight.nvstream.jni.MoonBridge;
 import com.limelight.preferences.PreferenceConfiguration;
 import com.limelight.ui.GameGestures;
 import com.limelight.ui.StreamView;
@@ -35,7 +34,6 @@ import android.content.ComponentName;
 import android.content.Context;
 import android.content.Intent;
 import android.content.ServiceConnection;
-import android.content.res.Configuration;
 import android.graphics.Point;
 import android.hardware.input.InputManager;
 import android.media.AudioManager;
@@ -61,8 +59,6 @@ import android.widget.FrameLayout;
 import android.view.inputmethod.InputMethodManager;
 import android.widget.Toast;
 
-import java.util.Locale;
-
 
 public class Game extends Activity implements SurfaceHolder.Callback,
     OnGenericMotionListener, OnTouchListener, NvConnectionListener, EvdevListener,
@@ -83,7 +79,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
     private ControllerHandler controllerHandler;
     private VirtualController virtualController;
-    private KeyboardTranslator keybTranslator;
 
     private PreferenceConfiguration prefConfig;
 
@@ -101,12 +96,10 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
     private ShortcutHelper shortcutHelper;
 
-    private EnhancedDecoderRenderer decoderRenderer;
+    private MediaCodecDecoderRenderer decoderRenderer;
 
     private WifiManager.WifiLock wifiLock;
 
-    private int drFlags = 0;
-
     private boolean connectedToUsbDriverService = false;
     private ServiceConnection usbDriverServiceConnection = new ServiceConnection() {
         @Override
@@ -173,10 +166,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
         // Read the stream preferences
         prefConfig = PreferenceConfiguration.readPreferences(this);
 
-        if (prefConfig.stretchVideo) {
-            drFlags |= VideoDecoderRenderer.FLAG_FILL_SCREEN;
-        }
-
         // Listen for events on the game surface
         streamView = (StreamView) findViewById(R.id.surfaceView);
         streamView.setOnGenericMotionListener(this);
@@ -236,8 +225,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
                 .setApp(new NvApp(appName, appId))
                 .setBitrate(prefConfig.bitrate * 1000)
                 .setEnableSops(prefConfig.enableSops)
-                .enableAdaptiveResolution((decoderRenderer.getCapabilities() &
-                        VideoDecoderRenderer.CAPABILITY_ADAPTIVE_RESOLUTION) != 0)
                 .enableLocalAudioPlayback(prefConfig.playHostAudio)
                 .setMaxPacketSize(remote ? 1024 : 1292)
                 .setRemote(remote)
@@ -249,7 +236,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
         // Initialize the connection
         conn = new NvConnection(host, uniqueId, Game.this, config, PlatformBinding.getCryptoProvider(this));
-        keybTranslator = new KeyboardTranslator(conn);
         controllerHandler = new ControllerHandler(this, conn, this, prefConfig.multiController, prefConfig.deadzonePercentage);
 
         InputManager inputManager = (InputManager) getSystemService(Context.INPUT_SERVICE);
@@ -438,7 +424,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
         }
 
         if (conn != null) {
-            VideoDecoderRenderer.VideoFormat videoFormat = conn.getActiveVideoFormat();
+            int videoFormat = conn.getActiveVideoFormat();
 
             displayedFailureDialog = true;
             stopConnection();
@@ -457,11 +443,11 @@ public class Game extends Activity implements SurfaceHolder.Callback,
             }
 
             // Add the video codec to the post-stream toast
-            if (message != null && videoFormat != VideoDecoderRenderer.VideoFormat.Unknown) {
+            if (message != null) {
-                if (videoFormat == VideoDecoderRenderer.VideoFormat.H265) {
+                if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) {
                     message += " [H.265]";
                 }
-                else {
+                else if (videoFormat == MoonBridge.VIDEO_FORMAT_H264) {
                     message += " [H.264]";
                 }
             }
@@ -584,7 +570,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
         if (!handled) {
             // Try the keyboard handler
-            short translated = keybTranslator.translate(event.getKeyCode());
+            short translated = KeyboardTranslator.translate(event.getKeyCode());
             if (translated == 0) {
                 return super.onKeyDown(keyCode, event);
             }
@@ -599,8 +585,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
                 return super.onKeyDown(keyCode, event);
             }
 
-            keybTranslator.sendKeyDown(translated,
+            conn.sendKeyboardInput(translated, KeyboardPacket.KEY_DOWN, getModifierState());
-                    getModifierState(event));
         }
 
         return true;
@@ -623,7 +608,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
         if (!handled) {
             // Try the keyboard handler
-            short translated = keybTranslator.translate(event.getKeyCode());
+            short translated = KeyboardTranslator.translate(event.getKeyCode());
             if (translated == 0) {
                 return super.onKeyUp(keyCode, event);
             }
@@ -637,8 +622,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
                 return super.onKeyUp(keyCode, event);
             }
 
-            keybTranslator.sendKeyUp(translated,
+            conn.sendKeyboardInput(translated, KeyboardPacket.KEY_UP, getModifierState(event));
-                    getModifierState(event));
         }
 
         return true;
@@ -880,14 +864,14 @@ public class Game extends Activity implements SurfaceHolder.Callback,
     }
 
     @Override
-    public void stageStarting(Stage stage) {
+    public void stageStarting(String stage) {
         if (spinner != null) {
-            spinner.setMessage(getResources().getString(R.string.conn_starting)+" "+stage.getName());
+            spinner.setMessage(getResources().getString(R.string.conn_starting)+" "+stage);
         }
     }
 
     @Override
-    public void stageComplete(Stage stage) {
+    public void stageComplete(String stage) {
     }
 
     private void stopConnection() {
@@ -904,7 +888,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
     }
 
     @Override
-    public void stageFailed(Stage stage) {
+    public void stageFailed(String stage, long errorCode) {
         if (spinner != null) {
             spinner.dismiss();
             spinner = null;
@@ -912,17 +896,19 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
         if (!displayedFailureDialog) {
             displayedFailureDialog = true;
+            LimeLog.severe(stage+" failed: "+errorCode);
+
             stopConnection();
             Dialog.displayDialog(this, getResources().getString(R.string.conn_error_title),
-                    getResources().getString(R.string.conn_error_msg)+" "+stage.getName(), true);
+                    getResources().getString(R.string.conn_error_msg)+" "+stage, true);
         }
     }
 
     @Override
-    public void connectionTerminated(Exception e) {
+    public void connectionTerminated(long errorCode) {
         if (!displayedFailureDialog) {
             displayedFailureDialog = true;
-            e.printStackTrace();
+            LimeLog.severe("Connection terminated: "+errorCode);
 
             stopConnection();
             Dialog.displayDialog(this, getResources().getString(R.string.conn_terminated_title),
@@ -984,8 +970,8 @@ public class Game extends Activity implements SurfaceHolder.Callback,
         if (!connected && !connecting) {
             connecting = true;
 
-            conn.start(PlatformBinding.getDeviceName(), holder, drFlags,
+            decoderRenderer.setRenderTarget(holder);
-                    PlatformBinding.getAudioRenderer(), decoderRenderer);
+            conn.start(PlatformBinding.getAudioRenderer(), decoderRenderer);
         }
     }
 
@@ -1044,17 +1030,17 @@ public class Game extends Activity implements SurfaceHolder.Callback,
 
     @Override
     public void keyboardEvent(boolean buttonDown, short keyCode) {
-        short keyMap = keybTranslator.translate(keyCode);
+        short keyMap = KeyboardTranslator.translate(keyCode);
         if (keyMap != 0) {
             if (handleSpecialKeys(keyMap, buttonDown)) {
                 return;
             }
 
             if (buttonDown) {
-                keybTranslator.sendKeyDown(keyMap, getModifierState());
+                conn.sendKeyboardInput(keyMap, KeyboardPacket.KEY_DOWN, getModifierState());
             }
             else {
-                keybTranslator.sendKeyUp(keyMap, getModifierState());
+                conn.sendKeyboardInput(keyMap, KeyboardPacket.KEY_UP, getModifierState());
             }
         }
     }

app/src/main/java/com/limelight/binding/audio/AndroidAudioRenderer.java

@@ -6,34 +6,31 @@ import android.media.AudioTrack;
 
 import com.limelight.LimeLog;
 import com.limelight.nvstream.av.audio.AudioRenderer;
+import com.limelight.nvstream.jni.MoonBridge;
 
 public class AndroidAudioRenderer implements AudioRenderer {
 
     private AudioTrack track;
 
     @Override
-    public boolean streamInitialized(int channelCount, int channelMask, int samplesPerFrame, int sampleRate) {
+    public void setup(int audioConfiguration) {
         int channelConfig;
         int bufferSize;
-        int bytesPerFrame = (samplesPerFrame * 2);
+        int bytesPerFrame;
 
-        switch (channelCount)
+        switch (audioConfiguration)
         {
-        case 1:
+            case MoonBridge.AUDIO_CONFIGURATION_STEREO:
-            channelConfig = AudioFormat.CHANNEL_OUT_MONO;
+                channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
-            break;
+                bytesPerFrame = 2 * 240 * 2;
-        case 2:
+                break;
-            channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
+            case MoonBridge.AUDIO_CONFIGURATION_51_SURROUND:
-            break;
+                channelConfig = AudioFormat.CHANNEL_OUT_5POINT1;
-        case 4:
+                bytesPerFrame = 6 * 240 * 2;
-            channelConfig = AudioFormat.CHANNEL_OUT_QUAD;
+                break;
-            break;
+            default:
-        case 6:
+                LimeLog.severe("Decoder returned unhandled channel count");
-            channelConfig = AudioFormat.CHANNEL_OUT_5POINT1;
+                return;
-            break;
-        default:
-            LimeLog.severe("Decoder returned unhandled channel count");
-            return false;
         }
 
         // We're not supposed to request less than the minimum
@@ -46,7 +43,7 @@ public class AndroidAudioRenderer implements AudioRenderer {
             bufferSize = bytesPerFrame * 2;
 
             track = new AudioTrack(AudioManager.STREAM_MUSIC,
-                    sampleRate,
+                    48000,
                     channelConfig,
                     AudioFormat.ENCODING_PCM_16BIT,
                     bufferSize,
@@ -61,7 +58,7 @@ public class AndroidAudioRenderer implements AudioRenderer {
             } catch (Exception ignored) {}
 
             // Now try the larger buffer size
-            bufferSize = Math.max(AudioTrack.getMinBufferSize(sampleRate,
+            bufferSize = Math.max(AudioTrack.getMinBufferSize(48000,
                             channelConfig,
                             AudioFormat.ENCODING_PCM_16BIT),
                     bytesPerFrame * 2);
@@ -70,7 +67,7 @@ public class AndroidAudioRenderer implements AudioRenderer {
             bufferSize = (((bufferSize + (bytesPerFrame - 1)) / bytesPerFrame) * bytesPerFrame);
 
             track = new AudioTrack(AudioManager.STREAM_MUSIC,
-                    sampleRate,
+                    48000,
                     channelConfig,
                     AudioFormat.ENCODING_PCM_16BIT,
                     bufferSize,
@@ -79,17 +76,15 @@ public class AndroidAudioRenderer implements AudioRenderer {
         }
 
         LimeLog.info("Audio track buffer size: "+bufferSize);
-
-        return true;
     }
 
     @Override
-    public void playDecodedAudio(byte[] audioData, int offset, int length) {
+    public void playDecodedAudio(byte[] audioData) {
-        track.write(audioData, offset, length);
+        track.write(audioData, 0, audioData.length);
     }
 
     @Override
-    public void streamClosing() {
+    public void cleanup() {
         if (track != null) {
             track.release();
         }

app/src/main/java/com/limelight/binding/input/KeyboardTranslator.java

@@ -3,14 +3,13 @@ package com.limelight.binding.input;
 import android.view.KeyEvent;
 
 import com.limelight.nvstream.NvConnection;
-import com.limelight.nvstream.input.KeycodeTranslator;
 
 /**
  * Class to translate a Android key code into the codes GFE is expecting
  * @author Diego Waxemberg
  * @author Cameron Gutman
  */
-public class KeyboardTranslator extends KeycodeTranslator {
+public class KeyboardTranslator {
 	
 	/**
 	 * GFE's prefix for every key code
@@ -59,21 +58,12 @@ public class KeyboardTranslator extends KeycodeTranslator {
 	public static final int VK_QUOTE = 222;
 	public static final int VK_PAUSE = 19;
 	
-	/**
-	 * Constructs a new translator for the specified connection
-	 * @param conn the connection to which the translated codes are sent
-	 */
-	public KeyboardTranslator(NvConnection conn) {
-		super(conn);
-	}
-	
 	/**
 	 * Translates the given keycode and returns the GFE keycode
 	 * @param keycode the code to be translated
 	 * @return a GFE keycode for the given keycode
 	 */
-	@Override
+	public static short translate(int keycode) {
-	public short translate(int keycode) {
 		int translated;
 		
 		/* There seems to be no clean mapping between Android key codes

app/src/main/java/com/limelight/binding/video/EnhancedDecoderRenderer.java

@@ -1,8 +0,0 @@
-package com.limelight.binding.video;
-
-import com.limelight.nvstream.av.video.VideoDecoderRenderer;
-
-public abstract class EnhancedDecoderRenderer extends VideoDecoderRenderer {
-    public abstract boolean isHevcSupported();
-    public abstract boolean isAvcSupported();
-}

app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java

@@ -2,17 +2,14 @@ package com.limelight.binding.video;
 
 import java.nio.ByteBuffer;
 import java.util.Locale;
-import java.util.concurrent.locks.LockSupport;
 
 import org.jcodec.codecs.h264.H264Utils;
 import org.jcodec.codecs.h264.io.model.SeqParameterSet;
 import org.jcodec.codecs.h264.io.model.VUIParameters;
 
 import com.limelight.LimeLog;
-import com.limelight.nvstream.av.ByteBufferDescriptor;
-import com.limelight.nvstream.av.DecodeUnit;
 import com.limelight.nvstream.av.video.VideoDecoderRenderer;
-import com.limelight.nvstream.av.video.VideoDepacketizer;
+import com.limelight.nvstream.jni.MoonBridge;
 import com.limelight.preferences.PreferenceConfiguration;
 
 import android.media.MediaCodec;
@@ -23,7 +20,7 @@ import android.media.MediaCodec.CodecException;
 import android.os.Build;
 import android.view.SurfaceHolder;
 
-public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
+public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
 
     private static final boolean USE_FRAME_RENDER_TIME = false;
 
@@ -36,24 +33,23 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
     private MediaCodec videoDecoder;
     private Thread rendererThread;
     private boolean needsSpsBitstreamFixup, isExynos4;
-    private VideoDepacketizer depacketizer;
     private boolean adaptivePlayback, directSubmit;
     private boolean constrainedHighProfile;
     private int initialWidth, initialHeight;
-    private VideoFormat videoFormat;
+    private int videoFormat;
+    private Object renderTarget;
 
     private boolean needsBaselineSpsHack;
     private SeqParameterSet savedSps;
 
     private long lastTimestampUs;
-    private long totalTimeMs;
     private long decoderTimeMs;
+    private long totalTimeMs;
     private int totalFrames;
 
     private int numSpsIn;
     private int numPpsIn;
     private int numVpsIn;
-    private int numIframeIn;
 
     private MediaCodecInfo findAvcDecoder() {
         MediaCodecInfo decoder = MediaCodecHelper.findProbableSafeDecoder("video/avc", MediaCodecInfo.CodecProfileLevel.AVCProfileHigh);
@@ -91,6 +87,10 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
         return decoderInfo;
     }
 
+    public void setRenderTarget(Object renderTarget) {
+        this.renderTarget = renderTarget;
+    }
+
     public MediaCodecDecoderRenderer(int videoFormat) {
         //dumpDecoders();
 
@@ -125,18 +125,16 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
         }
     }
 
-    @Override
     public boolean isHevcSupported() {
         return hevcDecoder != null;
     }
 
-    @Override
     public boolean isAvcSupported() {
         return avcDecoder != null;
     }
 
     @Override
-    public boolean setup(VideoDecoderRenderer.VideoFormat format, int width, int height, int redrawRate, Object renderTarget, int drFlags) {
+    public boolean setup(int format, int width, int height, int redrawRate) {
         this.initialWidth = width;
         this.initialHeight = height;
         this.videoFormat = format;
@@ -144,7 +142,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
         String mimeType;
         String selectedDecoderName;
 
-        if (videoFormat == VideoFormat.H264) {
+        if (videoFormat == MoonBridge.VIDEO_FORMAT_H264) {
             mimeType = "video/avc";
             selectedDecoderName = avcDecoder.getName();
 
@@ -171,7 +169,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
                 LimeLog.info("Decoder "+selectedDecoderName+" is on Exynos 4");
             }
         }
-        else if (videoFormat == VideoFormat.H265) {
+        else if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) {
             mimeType = "video/hevc";
             selectedDecoderName = hevcDecoder.getName();
 
@@ -229,6 +227,15 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
 
         LimeLog.info("Using codec "+selectedDecoderName+" for hardware decoding "+mimeType);
 
+        // Start the decoder
+        videoDecoder.start();
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
+            legacyInputBuffers = videoDecoder.getInputBuffers();
+        }
+
+        startRendererThread();
+
         return true;
     }
 
@@ -258,7 +265,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
         }
     }
 
-    private void startDirectSubmitRendererThread()
+    private void startRendererThread()
     {
         rendererThread = new Thread() {
             @Override
@@ -334,164 +341,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
         return index;
     }
 
-    private void startLegacyRendererThread()
+    // This method is used by the hack in Game, not called by the streaming core.
-    {
-        rendererThread = new Thread() {
-            @Override
-            public void run() {
-                BufferInfo info = new BufferInfo();
-                DecodeUnit du = null;
-                int inputIndex = -1;
-                long lastDuDequeueTime = 0;
-                while (!isInterrupted())
-                {
-                    // In order to get as much data to the decoder as early as possible,
-                    // try to submit up to 5 decode units at once without blocking.
-                    if (inputIndex == -1 && du == null) {
-                        try {
-                            for (int i = 0; i < 5; i++) {
-                                inputIndex = dequeueInputBuffer(false, false);
-                                du = depacketizer.pollNextDecodeUnit();
-                                if (du != null) {
-                                    lastDuDequeueTime = MediaCodecHelper.getMonotonicMillis();
-                                    notifyDuReceived(du);
-                                }
-
-                                // Stop if we can't get a DU or input buffer
-                                if (du == null || inputIndex == -1) {
-                                    break;
-                                }
-
-                                submitDecodeUnit(du, inputIndex);
-
-                                du = null;
-                                inputIndex = -1;
-                            }
-                        } catch (Exception e) {
-                            inputIndex = -1;
-                            handleDecoderException(e, null, 0);
-                        }
-                    }
-
-                    // Grab an input buffer if we don't have one already.
-                    // This way we can have one ready hopefully by the time
-                    // the depacketizer is done with this frame. It's important
-                    // that this can timeout because it's possible that we could exhaust
-                    // the decoder's input buffers and deadlocks because aren't pulling
-                    // frames out of the other end.
-                    if (inputIndex == -1) {
-                        try {
-                            // If we've got a DU waiting to be given to the decoder,
-                            // wait a full 3 ms for an input buffer. Otherwise
-                            // just see if we can get one immediately.
-                            inputIndex = dequeueInputBuffer(du != null, false);
-                        } catch (Exception e) {
-                            inputIndex = -1;
-                            handleDecoderException(e, null, 0);
-                        }
-                    }
-
-                    // Grab a decode unit if we don't have one already
-                    if (du == null) {
-                        du = depacketizer.pollNextDecodeUnit();
-                        if (du != null) {
-                            lastDuDequeueTime = MediaCodecHelper.getMonotonicMillis();
-                            notifyDuReceived(du);
-                        }
-                    }
-
-                    // If we've got both a decode unit and an input buffer, we'll
-                    // submit now. Otherwise, we wait until we have one.
-                    if (du != null && inputIndex >= 0) {
-                        long submissionTime = MediaCodecHelper.getMonotonicMillis();
-                        if (submissionTime - lastDuDequeueTime >= 20) {
-                            LimeLog.warning("Receiving an input buffer took too long: "+(submissionTime - lastDuDequeueTime)+" ms");
-                        }
-
-                        submitDecodeUnit(du, inputIndex);
-
-                        // DU and input buffer have both been consumed
-                        du = null;
-                        inputIndex = -1;
-                    }
-
-                    // Try to output a frame
-                    try {
-                        int outIndex = videoDecoder.dequeueOutputBuffer(info, 0);
-
-                        if (outIndex >= 0) {
-                            long presentationTimeUs = info.presentationTimeUs;
-                            int lastIndex = outIndex;
-
-                            // Get the last output buffer in the queue
-                            while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) {
-                                videoDecoder.releaseOutputBuffer(lastIndex, false);
-                                lastIndex = outIndex;
-                                presentationTimeUs = info.presentationTimeUs;
-                            }
-
-                            // Render the last buffer
-                            videoDecoder.releaseOutputBuffer(lastIndex, true);
-
-                            // Add delta time to the totals (excluding probable outliers)
-                            long delta = MediaCodecHelper.getMonotonicMillis()-(presentationTimeUs/1000);
-                            if (delta >= 0 && delta < 1000) {
-                                decoderTimeMs += delta;
-                                if (!USE_FRAME_RENDER_TIME) {
-                                    totalTimeMs += delta;
-                                }
-                            }
-                        } else {
-                            switch (outIndex) {
-                                case MediaCodec.INFO_TRY_AGAIN_LATER:
-                                    // Getting an input buffer may already block
-                                    // so don't park if we still need to do that
-                                    if (inputIndex >= 0) {
-                                        LockSupport.parkNanos(1);
-                                    }
-                                    break;
-                                case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
-                                    LimeLog.info("Output format changed");
-                                    LimeLog.info("New output Format: " + videoDecoder.getOutputFormat());
-                                    break;
-                                default:
-                                    break;
-                            }
-                        }
-                    } catch (Exception e) {
-                        handleDecoderException(e, null, 0);
-                    }
-                }
-            }
-        };
-        rendererThread.setName("Video - Renderer (MediaCodec)");
-        rendererThread.setPriority(Thread.MAX_PRIORITY);
-        rendererThread.start();
-    }
-
-    @SuppressWarnings("deprecation")
-    @Override
-    public boolean start(VideoDepacketizer depacketizer) {
-        this.depacketizer = depacketizer;
-
-        // Start the decoder
-        videoDecoder.start();
-
-        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
-            legacyInputBuffers = videoDecoder.getInputBuffers();
-        }
-
-        if (directSubmit) {
-            startDirectSubmitRendererThread();
-        }
-        else {
-            startLegacyRendererThread();
-        }
-
-        return true;
-    }
-
-    @Override
     public void stop() {
         if (rendererThread != null) {
             // Halt the rendering thread
@@ -500,13 +350,12 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
                 rendererThread.join();
             } catch (InterruptedException ignored) { }
         }
-
-        // We could stop the decoder here, but it seems to cause some problems
-        // so we'll just let release take care of it.
     }
 
     @Override
-    public void release() {
+    public void cleanup() {
+        stop();
+
         if (videoDecoder != null) {
             videoDecoder.release();
         }
@@ -570,7 +419,9 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
     }
 
     @SuppressWarnings("deprecation")
-    private void submitDecodeUnit(DecodeUnit decodeUnit, int inputBufferIndex) {
+    public int submitDecodeUnit(byte[] frameData) {
+        totalFrames++;
+
         long timestampUs = System.nanoTime() / 1000;
         if (timestampUs <= lastTimestampUs) {
             // We can't submit multiple buffers with the same timestamp
@@ -579,165 +430,155 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
         }
         lastTimestampUs = timestampUs;
 
+        int inputBufferIndex = dequeueInputBuffer(true, false);
         ByteBuffer buf = getEmptyInputBuffer(inputBufferIndex);
 
         int codecFlags = 0;
-        int decodeUnitFlags = decodeUnit.getFlags();
-        if ((decodeUnitFlags & DecodeUnit.DU_FLAG_CODEC_CONFIG) != 0) {
-            codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
-        }
-        if ((decodeUnitFlags & DecodeUnit.DU_FLAG_SYNC_FRAME) != 0) {
-            codecFlags |= MediaCodec.BUFFER_FLAG_SYNC_FRAME;
-            numIframeIn++;
-        }
-
         boolean needsSpsReplay = false;
 
-        if ((decodeUnitFlags & DecodeUnit.DU_FLAG_CODEC_CONFIG) != 0) {
+        // H264 SPS
-            ByteBufferDescriptor header = decodeUnit.getBufferHead();
+        if (frameData[4] == 0x67) {
+            numSpsIn++;
+            codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
 
-            // H264 SPS
+            ByteBuffer spsBuf = ByteBuffer.wrap(frameData);
-            if (header.data[header.offset+4] == 0x67) {
-                numSpsIn++;
 
-                ByteBuffer spsBuf = ByteBuffer.wrap(header.data);
+            // Skip to the start of the NALU data
+            spsBuf.position(5);
 
-                // Skip to the start of the NALU data
+            // The H264Utils.readSPS function safely handles
-                spsBuf.position(header.offset+5);
+            // Annex B NALUs (including NALUs with escape sequences)
+            SeqParameterSet sps = H264Utils.readSPS(spsBuf);
 
-                // The H264Utils.readSPS function safely handles
+            // Some decoders rely on H264 level to decide how many buffers are needed
-                // Annex B NALUs (including NALUs with escape sequences)
+            // Since we only need one frame buffered, we'll set the level as low as we can
-                SeqParameterSet sps = H264Utils.readSPS(spsBuf);
+            // for known resolution combinations
+            if (initialWidth == 1280 && initialHeight == 720) {
+                // Max 5 buffered frames at 1280x720x60
+                LimeLog.info("Patching level_idc to 32");
+                sps.level_idc = 32;
+            }
+            else if (initialWidth == 1920 && initialHeight == 1080) {
+                // Max 4 buffered frames at 1920x1080x64
+                LimeLog.info("Patching level_idc to 42");
+                sps.level_idc = 42;
+            }
+            else {
+                // Leave the profile alone (currently 5.0)
+            }
 
-                // Some decoders rely on H264 level to decide how many buffers are needed
+            // TI OMAP4 requires a reference frame count of 1 to decode successfully. Exynos 4
-                // Since we only need one frame buffered, we'll set the level as low as we can
+            // also requires this fixup.
-                // for known resolution combinations
+            //
-                if (initialWidth == 1280 && initialHeight == 720) {
+            // I'm doing this fixup for all devices because I haven't seen any devices that
-                    // Max 5 buffered frames at 1280x720x60
+            // this causes issues for. At worst, it seems to do nothing and at best it fixes
-                    LimeLog.info("Patching level_idc to 32");
+            // issues with video lag, hangs, and crashes.
-                    sps.level_idc = 32;
+            LimeLog.info("Patching num_ref_frames in SPS");
-                }
+            sps.num_ref_frames = 1;
-                else if (initialWidth == 1920 && initialHeight == 1080) {
+
-                    // Max 4 buffered frames at 1920x1080x64
+            // GFE 2.5.11 changed the SPS to add additional extensions
-                    LimeLog.info("Patching level_idc to 42");
+            // Some devices don't like these so we remove them here.
-                    sps.level_idc = 42;
+            sps.vuiParams.video_signal_type_present_flag = false;
+            sps.vuiParams.colour_description_present_flag = false;
+            sps.vuiParams.chroma_loc_info_present_flag = false;
+
+            if (needsSpsBitstreamFixup || isExynos4) {
+                // The SPS that comes in the current H264 bytestream doesn't set bitstream_restriction_flag
+                // or max_dec_frame_buffering which increases decoding latency on Tegra.
+
+                // GFE 2.5.11 started sending bitstream restrictions
+                if (sps.vuiParams.bitstreamRestriction == null) {
+                    LimeLog.info("Adding bitstream restrictions");
+                    sps.vuiParams.bitstreamRestriction = new VUIParameters.BitstreamRestriction();
+                    sps.vuiParams.bitstreamRestriction.motion_vectors_over_pic_boundaries_flag = true;
+                    sps.vuiParams.bitstreamRestriction.log2_max_mv_length_horizontal = 16;
+                    sps.vuiParams.bitstreamRestriction.log2_max_mv_length_vertical = 16;
+                    sps.vuiParams.bitstreamRestriction.num_reorder_frames = 0;
                 }
                 else {
-                    // Leave the profile alone (currently 5.0)
+                    LimeLog.info("Patching bitstream restrictions");
                 }
 
-                // TI OMAP4 requires a reference frame count of 1 to decode successfully. Exynos 4
+                // Some devices throw errors if max_dec_frame_buffering < num_ref_frames
-                // also requires this fixup.
+                sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering = sps.num_ref_frames;
-                //
-                // I'm doing this fixup for all devices because I haven't seen any devices that
-                // this causes issues for. At worst, it seems to do nothing and at best it fixes
-                // issues with video lag, hangs, and crashes.
-                LimeLog.info("Patching num_ref_frames in SPS");
-                sps.num_ref_frames = 1;
 
-                // GFE 2.5.11 changed the SPS to add additional extensions
+                // These values are the defaults for the fields, but they are more aggressive
-                // Some devices don't like these so we remove them here.
+                // than what GFE sends in 2.5.11, but it doesn't seem to cause picture problems.
-                sps.vuiParams.video_signal_type_present_flag = false;
+                sps.vuiParams.bitstreamRestriction.max_bytes_per_pic_denom = 2;
-                sps.vuiParams.colour_description_present_flag = false;
+                sps.vuiParams.bitstreamRestriction.max_bits_per_mb_denom = 1;
-                sps.vuiParams.chroma_loc_info_present_flag = false;
 
-                if (needsSpsBitstreamFixup || isExynos4) {
+                // log2_max_mv_length_horizontal and log2_max_mv_length_vertical are set to more
-                    // The SPS that comes in the current H264 bytestream doesn't set bitstream_restriction_flag
+                // conservative values by GFE 2.5.11. We'll let those values stand.
-                    // or max_dec_frame_buffering which increases decoding latency on Tegra.
-
-                    // GFE 2.5.11 started sending bitstream restrictions
-                    if (sps.vuiParams.bitstreamRestriction == null) {
-                        LimeLog.info("Adding bitstream restrictions");
-                        sps.vuiParams.bitstreamRestriction = new VUIParameters.BitstreamRestriction();
-                        sps.vuiParams.bitstreamRestriction.motion_vectors_over_pic_boundaries_flag = true;
-                        sps.vuiParams.bitstreamRestriction.log2_max_mv_length_horizontal = 16;
-                        sps.vuiParams.bitstreamRestriction.log2_max_mv_length_vertical = 16;
-                        sps.vuiParams.bitstreamRestriction.num_reorder_frames = 0;
-                    }
-                    else {
-                        LimeLog.info("Patching bitstream restrictions");
-                    }
-
-                    // Some devices throw errors if max_dec_frame_buffering < num_ref_frames
-                    sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering = sps.num_ref_frames;
-
-                    // These values are the defaults for the fields, but they are more aggressive
-                    // than what GFE sends in 2.5.11, but it doesn't seem to cause picture problems.
-                    sps.vuiParams.bitstreamRestriction.max_bytes_per_pic_denom = 2;
-                    sps.vuiParams.bitstreamRestriction.max_bits_per_mb_denom = 1;
-
-                    // log2_max_mv_length_horizontal and log2_max_mv_length_vertical are set to more
-                    // conservative values by GFE 2.5.11. We'll let those values stand.
-                }
-                else {
-                    // Devices that didn't/couldn't get bitstream restrictions before GFE 2.5.11
-                    // will continue to not receive them now
-                    sps.vuiParams.bitstreamRestriction = null;
-                }
-
-                // If we need to hack this SPS to say we're baseline, do so now
-                if (needsBaselineSpsHack) {
-                    LimeLog.info("Hacking SPS to baseline");
-                    sps.profile_idc = 66;
-                    savedSps = sps;
-                }
-
-                // Patch the SPS constraint flags
-                doProfileSpecificSpsPatching(sps);
-
-                // Write the annex B header
-                buf.put(header.data, header.offset, 5);
-
-                // The H264Utils.writeSPS function safely handles
-                // Annex B NALUs (including NALUs with escape sequences)
-                ByteBuffer escapedNalu = H264Utils.writeSPS(sps, header.length);
-                buf.put(escapedNalu);
-
-                queueInputBuffer(inputBufferIndex,
-                        0, buf.position(),
-                        timestampUs, codecFlags);
-
-                depacketizer.freeDecodeUnit(decodeUnit);
-                return;
-
-                // H264 PPS
-            } else if (header.data[header.offset+4] == 0x68) {
-                numPpsIn++;
-
-                if (needsBaselineSpsHack) {
-                    LimeLog.info("Saw PPS; disabling SPS hack");
-                    needsBaselineSpsHack = false;
-
-                    // Give the decoder the SPS again with the proper profile now
-                    needsSpsReplay = true;
-                }
             }
-            else if (header.data[header.offset+4] == 0x40) {
+            else {
-                numVpsIn++;
+                // Devices that didn't/couldn't get bitstream restrictions before GFE 2.5.11
+                // will continue to not receive them now
+                sps.vuiParams.bitstreamRestriction = null;
             }
-            else if (header.data[header.offset+4] == 0x42) {
+
-                numSpsIn++;
+            // If we need to hack this SPS to say we're baseline, do so now
+            if (needsBaselineSpsHack) {
+                LimeLog.info("Hacking SPS to baseline");
+                sps.profile_idc = 66;
+                savedSps = sps;
             }
-            else if (header.data[header.offset+4] == 0x44) {
+
-                numPpsIn++;
+            // Patch the SPS constraint flags
+            doProfileSpecificSpsPatching(sps);
+
+            // Write the annex B header
+            buf.put(frameData, 0, 5);
+
+            // The H264Utils.writeSPS function safely handles
+            // Annex B NALUs (including NALUs with escape sequences)
+            ByteBuffer escapedNalu = H264Utils.writeSPS(sps, frameData.length);
+            buf.put(escapedNalu);
+
+            queueInputBuffer(inputBufferIndex,
+                    0, buf.position(),
+                    timestampUs, codecFlags);
+
+            return MoonBridge.DR_OK;
+
+            // H264 PPS
+        } else if (frameData[4] == 0x68) {
+            numPpsIn++;
+            codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
+
+
+            if (needsBaselineSpsHack) {
+                LimeLog.info("Saw PPS; disabling SPS hack");
+                needsBaselineSpsHack = false;
+
+                // Give the decoder the SPS again with the proper profile now
+                needsSpsReplay = true;
             }
         }
+        else if (frameData[4] == 0x40) {
+            numVpsIn++;
+            codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
+        }
+        else if (frameData[4] == 0x42) {
+            numSpsIn++;
+            codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
+        }
+        else if (frameData[4] == 0x44) {
+            numPpsIn++;
+            codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
+        }
 
         // Copy data from our buffer list into the input buffer
-        for (ByteBufferDescriptor desc = decodeUnit.getBufferHead();
+        buf.put(frameData);
-             desc != null; desc = desc.nextDescriptor) {
-            buf.put(desc.data, desc.offset, desc.length);
-        }
 
         queueInputBuffer(inputBufferIndex,
-                0, decodeUnit.getDataLength(),
+                0, frameData.length,
                 timestampUs, codecFlags);
 
-        depacketizer.freeDecodeUnit(decodeUnit);
-
         if (needsSpsReplay) {
             replaySps();
         }
+
+        return MoonBridge.DR_OK;
     }
 
     private void replaySps() {
@@ -774,24 +615,12 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
     public int getCapabilities() {
         int caps = 0;
 
-        caps |= adaptivePlayback ?
-                VideoDecoderRenderer.CAPABILITY_ADAPTIVE_RESOLUTION : 0;
-
         caps |= directSubmit ?
-                VideoDecoderRenderer.CAPABILITY_DIRECT_SUBMIT : 0;
+                MoonBridge.CAPABILITY_DIRECT_SUBMIT : 0;
 
         return caps;
     }
 
-    @Override
-    public int getAverageDecoderLatency() {
-        if (totalFrames == 0) {
-            return 0;
-        }
-        return (int)(decoderTimeMs / totalFrames);
-    }
-
-    @Override
     public int getAverageEndToEndLatency() {
         if (totalFrames == 0) {
             return 0;
@@ -799,33 +628,11 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
         return (int)(totalTimeMs / totalFrames);
     }
 
-    private void notifyDuReceived(DecodeUnit du) {
+    public int getAverageDecoderLatency() {
-        long currentTime = MediaCodecHelper.getMonotonicMillis();
+        if (totalFrames == 0) {
-        long delta = currentTime-du.getReceiveTimestamp();
+            return 0;
-        if (delta >= 0 && delta < 1000) {
-            totalTimeMs += currentTime-du.getReceiveTimestamp();
-            totalFrames++;
-        }
-    }
-
-    @Override
-    public void directSubmitDecodeUnit(DecodeUnit du) {
-        int inputIndex = -1;
-
-        notifyDuReceived(du);
-
-        while (!Thread.currentThread().isInterrupted()) {
-            try {
-                inputIndex = dequeueInputBuffer(true, true);
-                break;
-            } catch (Exception e) {
-                handleDecoderException(e, null, 0);
-            }
-        }
-
-        if (inputIndex >= 0) {
-            submitDecodeUnit(du, inputIndex);
         }
+        return (int)(decoderTimeMs / totalFrames);
     }
 
     public class RendererException extends RuntimeException {
@@ -855,7 +662,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
             str += "AVC Decoder: "+((renderer.avcDecoder != null) ? renderer.avcDecoder.getName():"(none)")+"\n";
             str += "HEVC Decoder: "+((renderer.hevcDecoder != null) ? renderer.hevcDecoder.getName():"(none)")+"\n";
             str += "Initial video dimensions: "+renderer.initialWidth+"x"+renderer.initialHeight+"\n";
-            str += "In stats: "+renderer.numVpsIn+", "+renderer.numSpsIn+", "+renderer.numPpsIn+", "+renderer.numIframeIn+"\n";
+            str += "In stats: "+renderer.numVpsIn+", "+renderer.numSpsIn+", "+renderer.numPpsIn+"\n";
             str += "Total frames: "+renderer.totalFrames+"\n";
             str += "Average end-to-end client latency: "+getAverageEndToEndLatency()+"ms\n";
             str += "Average hardware decoder latency: "+getAverageDecoderLatency()+"ms\n";

app/src/main/java/com/limelight/utils/Vector2d.java

@@ -0,0 +1,47 @@
+package com.limelight.utils;
+
+public class Vector2d {
+	private float x;
+	private float y;
+	private double magnitude;
+	
+	public static final Vector2d ZERO = new Vector2d();
+	
+	public Vector2d() {
+		initialize(0, 0);
+	}
+	
+	public void initialize(float x, float y) {
+		this.x = x;
+		this.y = y;
+		this.magnitude = Math.sqrt(Math.pow(x, 2) + Math.pow(y, 2));
+	}
+	
+	public double getMagnitude() {
+		return magnitude;
+	}
+	
+	public void getNormalized(Vector2d vector) {
+		vector.initialize((float)(x / magnitude), (float)(y / magnitude));
+	}
+	
+	public void scalarMultiply(double factor) {
+		initialize((float)(x * factor), (float)(y * factor));
+	}
+	
+	public void setX(float x) {
+		initialize(x, this.y);
+	}
+	
+	public void setY(float y) {
+		initialize(this.x, y);
+	}
+	
+	public float getX() {
+		return x;
+	}
+	
+	public float getY() {
+		return y;
+	}
+}

moonlight-common

@@ -1 +1 @@
-Subproject commit 16818d41068fdd9005f2a9f159c72054766c008c
+Subproject commit f5b8743d693af4884e65f45243a8507fc598f40d