diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index ca91137e..5c5de4d2 100644 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -10,16 +10,15 @@ import com.limelight.binding.input.TouchContext; import com.limelight.binding.input.driver.UsbDriverService; import com.limelight.binding.input.evdev.EvdevListener; import com.limelight.binding.input.virtual_controller.VirtualController; -import com.limelight.binding.video.EnhancedDecoderRenderer; import com.limelight.binding.video.MediaCodecDecoderRenderer; import com.limelight.binding.video.MediaCodecHelper; import com.limelight.nvstream.NvConnection; import com.limelight.nvstream.NvConnectionListener; import com.limelight.nvstream.StreamConfiguration; -import com.limelight.nvstream.av.video.VideoDecoderRenderer; import com.limelight.nvstream.http.NvApp; import com.limelight.nvstream.input.KeyboardPacket; import com.limelight.nvstream.input.MouseButtonPacket; +import com.limelight.nvstream.jni.MoonBridge; import com.limelight.preferences.PreferenceConfiguration; import com.limelight.ui.GameGestures; import com.limelight.ui.StreamView; @@ -35,7 +34,6 @@ import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.ServiceConnection; -import android.content.res.Configuration; import android.graphics.Point; import android.hardware.input.InputManager; import android.media.AudioManager; @@ -61,8 +59,6 @@ import android.widget.FrameLayout; import android.view.inputmethod.InputMethodManager; import android.widget.Toast; -import java.util.Locale; - public class Game extends Activity implements SurfaceHolder.Callback, OnGenericMotionListener, OnTouchListener, NvConnectionListener, EvdevListener, @@ -83,7 +79,6 @@ public class Game extends Activity implements SurfaceHolder.Callback, private ControllerHandler controllerHandler; private VirtualController virtualController; - private KeyboardTranslator keybTranslator; private PreferenceConfiguration prefConfig; @@ -101,12 +96,10 @@ public class Game extends Activity implements SurfaceHolder.Callback, private ShortcutHelper shortcutHelper; - private EnhancedDecoderRenderer decoderRenderer; + private MediaCodecDecoderRenderer decoderRenderer; private WifiManager.WifiLock wifiLock; - private int drFlags = 0; - private boolean connectedToUsbDriverService = false; private ServiceConnection usbDriverServiceConnection = new ServiceConnection() { @Override @@ -173,10 +166,6 @@ public class Game extends Activity implements SurfaceHolder.Callback, // Read the stream preferences prefConfig = PreferenceConfiguration.readPreferences(this); - if (prefConfig.stretchVideo) { - drFlags |= VideoDecoderRenderer.FLAG_FILL_SCREEN; - } - // Listen for events on the game surface streamView = (StreamView) findViewById(R.id.surfaceView); streamView.setOnGenericMotionListener(this); @@ -236,8 +225,6 @@ public class Game extends Activity implements SurfaceHolder.Callback, .setApp(new NvApp(appName, appId)) .setBitrate(prefConfig.bitrate * 1000) .setEnableSops(prefConfig.enableSops) - .enableAdaptiveResolution((decoderRenderer.getCapabilities() & - VideoDecoderRenderer.CAPABILITY_ADAPTIVE_RESOLUTION) != 0) .enableLocalAudioPlayback(prefConfig.playHostAudio) .setMaxPacketSize(remote ? 1024 : 1292) .setRemote(remote) @@ -249,7 +236,6 @@ public class Game extends Activity implements SurfaceHolder.Callback, // Initialize the connection conn = new NvConnection(host, uniqueId, Game.this, config, PlatformBinding.getCryptoProvider(this)); - keybTranslator = new KeyboardTranslator(conn); controllerHandler = new ControllerHandler(this, conn, this, prefConfig.multiController, prefConfig.deadzonePercentage); InputManager inputManager = (InputManager) getSystemService(Context.INPUT_SERVICE); @@ -438,7 +424,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, } if (conn != null) { - VideoDecoderRenderer.VideoFormat videoFormat = conn.getActiveVideoFormat(); + int videoFormat = conn.getActiveVideoFormat(); displayedFailureDialog = true; stopConnection(); @@ -457,11 +443,11 @@ public class Game extends Activity implements SurfaceHolder.Callback, } // Add the video codec to the post-stream toast - if (message != null && videoFormat != VideoDecoderRenderer.VideoFormat.Unknown) { - if (videoFormat == VideoDecoderRenderer.VideoFormat.H265) { + if (message != null) { + if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) { message += " [H.265]"; } - else { + else if (videoFormat == MoonBridge.VIDEO_FORMAT_H264) { message += " [H.264]"; } } @@ -584,7 +570,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, if (!handled) { // Try the keyboard handler - short translated = keybTranslator.translate(event.getKeyCode()); + short translated = KeyboardTranslator.translate(event.getKeyCode()); if (translated == 0) { return super.onKeyDown(keyCode, event); } @@ -599,8 +585,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, return super.onKeyDown(keyCode, event); } - keybTranslator.sendKeyDown(translated, - getModifierState(event)); + conn.sendKeyboardInput(translated, KeyboardPacket.KEY_DOWN, getModifierState()); } return true; @@ -623,7 +608,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, if (!handled) { // Try the keyboard handler - short translated = keybTranslator.translate(event.getKeyCode()); + short translated = KeyboardTranslator.translate(event.getKeyCode()); if (translated == 0) { return super.onKeyUp(keyCode, event); } @@ -637,8 +622,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, return super.onKeyUp(keyCode, event); } - keybTranslator.sendKeyUp(translated, - getModifierState(event)); + conn.sendKeyboardInput(translated, KeyboardPacket.KEY_UP, getModifierState(event)); } return true; @@ -880,14 +864,14 @@ public class Game extends Activity implements SurfaceHolder.Callback, } @Override - public void stageStarting(Stage stage) { + public void stageStarting(String stage) { if (spinner != null) { - spinner.setMessage(getResources().getString(R.string.conn_starting)+" "+stage.getName()); + spinner.setMessage(getResources().getString(R.string.conn_starting)+" "+stage); } } @Override - public void stageComplete(Stage stage) { + public void stageComplete(String stage) { } private void stopConnection() { @@ -904,7 +888,7 @@ public class Game extends Activity implements SurfaceHolder.Callback, } @Override - public void stageFailed(Stage stage) { + public void stageFailed(String stage, long errorCode) { if (spinner != null) { spinner.dismiss(); spinner = null; @@ -912,17 +896,19 @@ public class Game extends Activity implements SurfaceHolder.Callback, if (!displayedFailureDialog) { displayedFailureDialog = true; + LimeLog.severe(stage+" failed: "+errorCode); + stopConnection(); Dialog.displayDialog(this, getResources().getString(R.string.conn_error_title), - getResources().getString(R.string.conn_error_msg)+" "+stage.getName(), true); + getResources().getString(R.string.conn_error_msg)+" "+stage, true); } } @Override - public void connectionTerminated(Exception e) { + public void connectionTerminated(long errorCode) { if (!displayedFailureDialog) { displayedFailureDialog = true; - e.printStackTrace(); + LimeLog.severe("Connection terminated: "+errorCode); stopConnection(); Dialog.displayDialog(this, getResources().getString(R.string.conn_terminated_title), @@ -984,8 +970,8 @@ public class Game extends Activity implements SurfaceHolder.Callback, if (!connected && !connecting) { connecting = true; - conn.start(PlatformBinding.getDeviceName(), holder, drFlags, - PlatformBinding.getAudioRenderer(), decoderRenderer); + decoderRenderer.setRenderTarget(holder); + conn.start(PlatformBinding.getAudioRenderer(), decoderRenderer); } } @@ -1044,17 +1030,17 @@ public class Game extends Activity implements SurfaceHolder.Callback, @Override public void keyboardEvent(boolean buttonDown, short keyCode) { - short keyMap = keybTranslator.translate(keyCode); + short keyMap = KeyboardTranslator.translate(keyCode); if (keyMap != 0) { if (handleSpecialKeys(keyMap, buttonDown)) { return; } if (buttonDown) { - keybTranslator.sendKeyDown(keyMap, getModifierState()); + conn.sendKeyboardInput(keyMap, KeyboardPacket.KEY_DOWN, getModifierState()); } else { - keybTranslator.sendKeyUp(keyMap, getModifierState()); + conn.sendKeyboardInput(keyMap, KeyboardPacket.KEY_UP, getModifierState()); } } } diff --git a/app/src/main/java/com/limelight/binding/audio/AndroidAudioRenderer.java b/app/src/main/java/com/limelight/binding/audio/AndroidAudioRenderer.java index 3b5621e2..017a9729 100644 --- a/app/src/main/java/com/limelight/binding/audio/AndroidAudioRenderer.java +++ b/app/src/main/java/com/limelight/binding/audio/AndroidAudioRenderer.java @@ -6,34 +6,31 @@ import android.media.AudioTrack; import com.limelight.LimeLog; import com.limelight.nvstream.av.audio.AudioRenderer; +import com.limelight.nvstream.jni.MoonBridge; public class AndroidAudioRenderer implements AudioRenderer { private AudioTrack track; @Override - public boolean streamInitialized(int channelCount, int channelMask, int samplesPerFrame, int sampleRate) { + public void setup(int audioConfiguration) { int channelConfig; int bufferSize; - int bytesPerFrame = (samplesPerFrame * 2); + int bytesPerFrame; - switch (channelCount) + switch (audioConfiguration) { - case 1: - channelConfig = AudioFormat.CHANNEL_OUT_MONO; - break; - case 2: - channelConfig = AudioFormat.CHANNEL_OUT_STEREO; - break; - case 4: - channelConfig = AudioFormat.CHANNEL_OUT_QUAD; - break; - case 6: - channelConfig = AudioFormat.CHANNEL_OUT_5POINT1; - break; - default: - LimeLog.severe("Decoder returned unhandled channel count"); - return false; + case MoonBridge.AUDIO_CONFIGURATION_STEREO: + channelConfig = AudioFormat.CHANNEL_OUT_STEREO; + bytesPerFrame = 2 * 240 * 2; + break; + case MoonBridge.AUDIO_CONFIGURATION_51_SURROUND: + channelConfig = AudioFormat.CHANNEL_OUT_5POINT1; + bytesPerFrame = 6 * 240 * 2; + break; + default: + LimeLog.severe("Decoder returned unhandled channel count"); + return; } // We're not supposed to request less than the minimum @@ -46,7 +43,7 @@ public class AndroidAudioRenderer implements AudioRenderer { bufferSize = bytesPerFrame * 2; track = new AudioTrack(AudioManager.STREAM_MUSIC, - sampleRate, + 48000, channelConfig, AudioFormat.ENCODING_PCM_16BIT, bufferSize, @@ -61,7 +58,7 @@ public class AndroidAudioRenderer implements AudioRenderer { } catch (Exception ignored) {} // Now try the larger buffer size - bufferSize = Math.max(AudioTrack.getMinBufferSize(sampleRate, + bufferSize = Math.max(AudioTrack.getMinBufferSize(48000, channelConfig, AudioFormat.ENCODING_PCM_16BIT), bytesPerFrame * 2); @@ -70,7 +67,7 @@ public class AndroidAudioRenderer implements AudioRenderer { bufferSize = (((bufferSize + (bytesPerFrame - 1)) / bytesPerFrame) * bytesPerFrame); track = new AudioTrack(AudioManager.STREAM_MUSIC, - sampleRate, + 48000, channelConfig, AudioFormat.ENCODING_PCM_16BIT, bufferSize, @@ -79,17 +76,15 @@ public class AndroidAudioRenderer implements AudioRenderer { } LimeLog.info("Audio track buffer size: "+bufferSize); - - return true; } @Override - public void playDecodedAudio(byte[] audioData, int offset, int length) { - track.write(audioData, offset, length); + public void playDecodedAudio(byte[] audioData) { + track.write(audioData, 0, audioData.length); } @Override - public void streamClosing() { + public void cleanup() { if (track != null) { track.release(); } diff --git a/app/src/main/java/com/limelight/binding/input/KeyboardTranslator.java b/app/src/main/java/com/limelight/binding/input/KeyboardTranslator.java index f087215f..839cb500 100644 --- a/app/src/main/java/com/limelight/binding/input/KeyboardTranslator.java +++ b/app/src/main/java/com/limelight/binding/input/KeyboardTranslator.java @@ -3,14 +3,13 @@ package com.limelight.binding.input; import android.view.KeyEvent; import com.limelight.nvstream.NvConnection; -import com.limelight.nvstream.input.KeycodeTranslator; /** * Class to translate a Android key code into the codes GFE is expecting * @author Diego Waxemberg * @author Cameron Gutman */ -public class KeyboardTranslator extends KeycodeTranslator { +public class KeyboardTranslator { /** * GFE's prefix for every key code @@ -59,21 +58,12 @@ public class KeyboardTranslator extends KeycodeTranslator { public static final int VK_QUOTE = 222; public static final int VK_PAUSE = 19; - /** - * Constructs a new translator for the specified connection - * @param conn the connection to which the translated codes are sent - */ - public KeyboardTranslator(NvConnection conn) { - super(conn); - } - /** * Translates the given keycode and returns the GFE keycode * @param keycode the code to be translated * @return a GFE keycode for the given keycode */ - @Override - public short translate(int keycode) { + public static short translate(int keycode) { int translated; /* There seems to be no clean mapping between Android key codes diff --git a/app/src/main/java/com/limelight/binding/video/EnhancedDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/EnhancedDecoderRenderer.java deleted file mode 100644 index 7378321f..00000000 --- a/app/src/main/java/com/limelight/binding/video/EnhancedDecoderRenderer.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.limelight.binding.video; - -import com.limelight.nvstream.av.video.VideoDecoderRenderer; - -public abstract class EnhancedDecoderRenderer extends VideoDecoderRenderer { - public abstract boolean isHevcSupported(); - public abstract boolean isAvcSupported(); -} diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index 6c858dae..14291739 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -2,17 +2,14 @@ package com.limelight.binding.video; import java.nio.ByteBuffer; import java.util.Locale; -import java.util.concurrent.locks.LockSupport; import org.jcodec.codecs.h264.H264Utils; import org.jcodec.codecs.h264.io.model.SeqParameterSet; import org.jcodec.codecs.h264.io.model.VUIParameters; import com.limelight.LimeLog; -import com.limelight.nvstream.av.ByteBufferDescriptor; -import com.limelight.nvstream.av.DecodeUnit; import com.limelight.nvstream.av.video.VideoDecoderRenderer; -import com.limelight.nvstream.av.video.VideoDepacketizer; +import com.limelight.nvstream.jni.MoonBridge; import com.limelight.preferences.PreferenceConfiguration; import android.media.MediaCodec; @@ -23,7 +20,7 @@ import android.media.MediaCodec.CodecException; import android.os.Build; import android.view.SurfaceHolder; -public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { +public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { private static final boolean USE_FRAME_RENDER_TIME = false; @@ -36,24 +33,23 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { private MediaCodec videoDecoder; private Thread rendererThread; private boolean needsSpsBitstreamFixup, isExynos4; - private VideoDepacketizer depacketizer; private boolean adaptivePlayback, directSubmit; private boolean constrainedHighProfile; private int initialWidth, initialHeight; - private VideoFormat videoFormat; + private int videoFormat; + private Object renderTarget; private boolean needsBaselineSpsHack; private SeqParameterSet savedSps; private long lastTimestampUs; - private long totalTimeMs; private long decoderTimeMs; + private long totalTimeMs; private int totalFrames; private int numSpsIn; private int numPpsIn; private int numVpsIn; - private int numIframeIn; private MediaCodecInfo findAvcDecoder() { MediaCodecInfo decoder = MediaCodecHelper.findProbableSafeDecoder("video/avc", MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); @@ -91,6 +87,10 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { return decoderInfo; } + public void setRenderTarget(Object renderTarget) { + this.renderTarget = renderTarget; + } + public MediaCodecDecoderRenderer(int videoFormat) { //dumpDecoders(); @@ -125,18 +125,16 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { } } - @Override public boolean isHevcSupported() { return hevcDecoder != null; } - @Override public boolean isAvcSupported() { return avcDecoder != null; } @Override - public boolean setup(VideoDecoderRenderer.VideoFormat format, int width, int height, int redrawRate, Object renderTarget, int drFlags) { + public boolean setup(int format, int width, int height, int redrawRate) { this.initialWidth = width; this.initialHeight = height; this.videoFormat = format; @@ -144,7 +142,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { String mimeType; String selectedDecoderName; - if (videoFormat == VideoFormat.H264) { + if (videoFormat == MoonBridge.VIDEO_FORMAT_H264) { mimeType = "video/avc"; selectedDecoderName = avcDecoder.getName(); @@ -171,7 +169,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { LimeLog.info("Decoder "+selectedDecoderName+" is on Exynos 4"); } } - else if (videoFormat == VideoFormat.H265) { + else if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) { mimeType = "video/hevc"; selectedDecoderName = hevcDecoder.getName(); @@ -229,6 +227,15 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { LimeLog.info("Using codec "+selectedDecoderName+" for hardware decoding "+mimeType); + // Start the decoder + videoDecoder.start(); + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + legacyInputBuffers = videoDecoder.getInputBuffers(); + } + + startRendererThread(); + return true; } @@ -258,7 +265,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { } } - private void startDirectSubmitRendererThread() + private void startRendererThread() { rendererThread = new Thread() { @Override @@ -334,164 +341,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { return index; } - private void startLegacyRendererThread() - { - rendererThread = new Thread() { - @Override - public void run() { - BufferInfo info = new BufferInfo(); - DecodeUnit du = null; - int inputIndex = -1; - long lastDuDequeueTime = 0; - while (!isInterrupted()) - { - // In order to get as much data to the decoder as early as possible, - // try to submit up to 5 decode units at once without blocking. - if (inputIndex == -1 && du == null) { - try { - for (int i = 0; i < 5; i++) { - inputIndex = dequeueInputBuffer(false, false); - du = depacketizer.pollNextDecodeUnit(); - if (du != null) { - lastDuDequeueTime = MediaCodecHelper.getMonotonicMillis(); - notifyDuReceived(du); - } - - // Stop if we can't get a DU or input buffer - if (du == null || inputIndex == -1) { - break; - } - - submitDecodeUnit(du, inputIndex); - - du = null; - inputIndex = -1; - } - } catch (Exception e) { - inputIndex = -1; - handleDecoderException(e, null, 0); - } - } - - // Grab an input buffer if we don't have one already. - // This way we can have one ready hopefully by the time - // the depacketizer is done with this frame. It's important - // that this can timeout because it's possible that we could exhaust - // the decoder's input buffers and deadlocks because aren't pulling - // frames out of the other end. - if (inputIndex == -1) { - try { - // If we've got a DU waiting to be given to the decoder, - // wait a full 3 ms for an input buffer. Otherwise - // just see if we can get one immediately. - inputIndex = dequeueInputBuffer(du != null, false); - } catch (Exception e) { - inputIndex = -1; - handleDecoderException(e, null, 0); - } - } - - // Grab a decode unit if we don't have one already - if (du == null) { - du = depacketizer.pollNextDecodeUnit(); - if (du != null) { - lastDuDequeueTime = MediaCodecHelper.getMonotonicMillis(); - notifyDuReceived(du); - } - } - - // If we've got both a decode unit and an input buffer, we'll - // submit now. Otherwise, we wait until we have one. - if (du != null && inputIndex >= 0) { - long submissionTime = MediaCodecHelper.getMonotonicMillis(); - if (submissionTime - lastDuDequeueTime >= 20) { - LimeLog.warning("Receiving an input buffer took too long: "+(submissionTime - lastDuDequeueTime)+" ms"); - } - - submitDecodeUnit(du, inputIndex); - - // DU and input buffer have both been consumed - du = null; - inputIndex = -1; - } - - // Try to output a frame - try { - int outIndex = videoDecoder.dequeueOutputBuffer(info, 0); - - if (outIndex >= 0) { - long presentationTimeUs = info.presentationTimeUs; - int lastIndex = outIndex; - - // Get the last output buffer in the queue - while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) { - videoDecoder.releaseOutputBuffer(lastIndex, false); - lastIndex = outIndex; - presentationTimeUs = info.presentationTimeUs; - } - - // Render the last buffer - videoDecoder.releaseOutputBuffer(lastIndex, true); - - // Add delta time to the totals (excluding probable outliers) - long delta = MediaCodecHelper.getMonotonicMillis()-(presentationTimeUs/1000); - if (delta >= 0 && delta < 1000) { - decoderTimeMs += delta; - if (!USE_FRAME_RENDER_TIME) { - totalTimeMs += delta; - } - } - } else { - switch (outIndex) { - case MediaCodec.INFO_TRY_AGAIN_LATER: - // Getting an input buffer may already block - // so don't park if we still need to do that - if (inputIndex >= 0) { - LockSupport.parkNanos(1); - } - break; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - LimeLog.info("Output format changed"); - LimeLog.info("New output Format: " + videoDecoder.getOutputFormat()); - break; - default: - break; - } - } - } catch (Exception e) { - handleDecoderException(e, null, 0); - } - } - } - }; - rendererThread.setName("Video - Renderer (MediaCodec)"); - rendererThread.setPriority(Thread.MAX_PRIORITY); - rendererThread.start(); - } - - @SuppressWarnings("deprecation") - @Override - public boolean start(VideoDepacketizer depacketizer) { - this.depacketizer = depacketizer; - - // Start the decoder - videoDecoder.start(); - - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { - legacyInputBuffers = videoDecoder.getInputBuffers(); - } - - if (directSubmit) { - startDirectSubmitRendererThread(); - } - else { - startLegacyRendererThread(); - } - - return true; - } - - @Override + // This method is used by the hack in Game, not called by the streaming core. public void stop() { if (rendererThread != null) { // Halt the rendering thread @@ -500,13 +350,12 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { rendererThread.join(); } catch (InterruptedException ignored) { } } - - // We could stop the decoder here, but it seems to cause some problems - // so we'll just let release take care of it. } @Override - public void release() { + public void cleanup() { + stop(); + if (videoDecoder != null) { videoDecoder.release(); } @@ -570,7 +419,9 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { } @SuppressWarnings("deprecation") - private void submitDecodeUnit(DecodeUnit decodeUnit, int inputBufferIndex) { + public int submitDecodeUnit(byte[] frameData) { + totalFrames++; + long timestampUs = System.nanoTime() / 1000; if (timestampUs <= lastTimestampUs) { // We can't submit multiple buffers with the same timestamp @@ -579,165 +430,155 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { } lastTimestampUs = timestampUs; + int inputBufferIndex = dequeueInputBuffer(true, false); ByteBuffer buf = getEmptyInputBuffer(inputBufferIndex); int codecFlags = 0; - int decodeUnitFlags = decodeUnit.getFlags(); - if ((decodeUnitFlags & DecodeUnit.DU_FLAG_CODEC_CONFIG) != 0) { - codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; - } - if ((decodeUnitFlags & DecodeUnit.DU_FLAG_SYNC_FRAME) != 0) { - codecFlags |= MediaCodec.BUFFER_FLAG_SYNC_FRAME; - numIframeIn++; - } - boolean needsSpsReplay = false; - if ((decodeUnitFlags & DecodeUnit.DU_FLAG_CODEC_CONFIG) != 0) { - ByteBufferDescriptor header = decodeUnit.getBufferHead(); + // H264 SPS + if (frameData[4] == 0x67) { + numSpsIn++; + codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; - // H264 SPS - if (header.data[header.offset+4] == 0x67) { - numSpsIn++; + ByteBuffer spsBuf = ByteBuffer.wrap(frameData); - ByteBuffer spsBuf = ByteBuffer.wrap(header.data); + // Skip to the start of the NALU data + spsBuf.position(5); - // Skip to the start of the NALU data - spsBuf.position(header.offset+5); + // The H264Utils.readSPS function safely handles + // Annex B NALUs (including NALUs with escape sequences) + SeqParameterSet sps = H264Utils.readSPS(spsBuf); - // The H264Utils.readSPS function safely handles - // Annex B NALUs (including NALUs with escape sequences) - SeqParameterSet sps = H264Utils.readSPS(spsBuf); + // Some decoders rely on H264 level to decide how many buffers are needed + // Since we only need one frame buffered, we'll set the level as low as we can + // for known resolution combinations + if (initialWidth == 1280 && initialHeight == 720) { + // Max 5 buffered frames at 1280x720x60 + LimeLog.info("Patching level_idc to 32"); + sps.level_idc = 32; + } + else if (initialWidth == 1920 && initialHeight == 1080) { + // Max 4 buffered frames at 1920x1080x64 + LimeLog.info("Patching level_idc to 42"); + sps.level_idc = 42; + } + else { + // Leave the profile alone (currently 5.0) + } - // Some decoders rely on H264 level to decide how many buffers are needed - // Since we only need one frame buffered, we'll set the level as low as we can - // for known resolution combinations - if (initialWidth == 1280 && initialHeight == 720) { - // Max 5 buffered frames at 1280x720x60 - LimeLog.info("Patching level_idc to 32"); - sps.level_idc = 32; - } - else if (initialWidth == 1920 && initialHeight == 1080) { - // Max 4 buffered frames at 1920x1080x64 - LimeLog.info("Patching level_idc to 42"); - sps.level_idc = 42; + // TI OMAP4 requires a reference frame count of 1 to decode successfully. Exynos 4 + // also requires this fixup. + // + // I'm doing this fixup for all devices because I haven't seen any devices that + // this causes issues for. At worst, it seems to do nothing and at best it fixes + // issues with video lag, hangs, and crashes. + LimeLog.info("Patching num_ref_frames in SPS"); + sps.num_ref_frames = 1; + + // GFE 2.5.11 changed the SPS to add additional extensions + // Some devices don't like these so we remove them here. + sps.vuiParams.video_signal_type_present_flag = false; + sps.vuiParams.colour_description_present_flag = false; + sps.vuiParams.chroma_loc_info_present_flag = false; + + if (needsSpsBitstreamFixup || isExynos4) { + // The SPS that comes in the current H264 bytestream doesn't set bitstream_restriction_flag + // or max_dec_frame_buffering which increases decoding latency on Tegra. + + // GFE 2.5.11 started sending bitstream restrictions + if (sps.vuiParams.bitstreamRestriction == null) { + LimeLog.info("Adding bitstream restrictions"); + sps.vuiParams.bitstreamRestriction = new VUIParameters.BitstreamRestriction(); + sps.vuiParams.bitstreamRestriction.motion_vectors_over_pic_boundaries_flag = true; + sps.vuiParams.bitstreamRestriction.log2_max_mv_length_horizontal = 16; + sps.vuiParams.bitstreamRestriction.log2_max_mv_length_vertical = 16; + sps.vuiParams.bitstreamRestriction.num_reorder_frames = 0; } else { - // Leave the profile alone (currently 5.0) + LimeLog.info("Patching bitstream restrictions"); } - // TI OMAP4 requires a reference frame count of 1 to decode successfully. Exynos 4 - // also requires this fixup. - // - // I'm doing this fixup for all devices because I haven't seen any devices that - // this causes issues for. At worst, it seems to do nothing and at best it fixes - // issues with video lag, hangs, and crashes. - LimeLog.info("Patching num_ref_frames in SPS"); - sps.num_ref_frames = 1; + // Some devices throw errors if max_dec_frame_buffering < num_ref_frames + sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering = sps.num_ref_frames; - // GFE 2.5.11 changed the SPS to add additional extensions - // Some devices don't like these so we remove them here. - sps.vuiParams.video_signal_type_present_flag = false; - sps.vuiParams.colour_description_present_flag = false; - sps.vuiParams.chroma_loc_info_present_flag = false; + // These values are the defaults for the fields, but they are more aggressive + // than what GFE sends in 2.5.11, but it doesn't seem to cause picture problems. + sps.vuiParams.bitstreamRestriction.max_bytes_per_pic_denom = 2; + sps.vuiParams.bitstreamRestriction.max_bits_per_mb_denom = 1; - if (needsSpsBitstreamFixup || isExynos4) { - // The SPS that comes in the current H264 bytestream doesn't set bitstream_restriction_flag - // or max_dec_frame_buffering which increases decoding latency on Tegra. - - // GFE 2.5.11 started sending bitstream restrictions - if (sps.vuiParams.bitstreamRestriction == null) { - LimeLog.info("Adding bitstream restrictions"); - sps.vuiParams.bitstreamRestriction = new VUIParameters.BitstreamRestriction(); - sps.vuiParams.bitstreamRestriction.motion_vectors_over_pic_boundaries_flag = true; - sps.vuiParams.bitstreamRestriction.log2_max_mv_length_horizontal = 16; - sps.vuiParams.bitstreamRestriction.log2_max_mv_length_vertical = 16; - sps.vuiParams.bitstreamRestriction.num_reorder_frames = 0; - } - else { - LimeLog.info("Patching bitstream restrictions"); - } - - // Some devices throw errors if max_dec_frame_buffering < num_ref_frames - sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering = sps.num_ref_frames; - - // These values are the defaults for the fields, but they are more aggressive - // than what GFE sends in 2.5.11, but it doesn't seem to cause picture problems. - sps.vuiParams.bitstreamRestriction.max_bytes_per_pic_denom = 2; - sps.vuiParams.bitstreamRestriction.max_bits_per_mb_denom = 1; - - // log2_max_mv_length_horizontal and log2_max_mv_length_vertical are set to more - // conservative values by GFE 2.5.11. We'll let those values stand. - } - else { - // Devices that didn't/couldn't get bitstream restrictions before GFE 2.5.11 - // will continue to not receive them now - sps.vuiParams.bitstreamRestriction = null; - } - - // If we need to hack this SPS to say we're baseline, do so now - if (needsBaselineSpsHack) { - LimeLog.info("Hacking SPS to baseline"); - sps.profile_idc = 66; - savedSps = sps; - } - - // Patch the SPS constraint flags - doProfileSpecificSpsPatching(sps); - - // Write the annex B header - buf.put(header.data, header.offset, 5); - - // The H264Utils.writeSPS function safely handles - // Annex B NALUs (including NALUs with escape sequences) - ByteBuffer escapedNalu = H264Utils.writeSPS(sps, header.length); - buf.put(escapedNalu); - - queueInputBuffer(inputBufferIndex, - 0, buf.position(), - timestampUs, codecFlags); - - depacketizer.freeDecodeUnit(decodeUnit); - return; - - // H264 PPS - } else if (header.data[header.offset+4] == 0x68) { - numPpsIn++; - - if (needsBaselineSpsHack) { - LimeLog.info("Saw PPS; disabling SPS hack"); - needsBaselineSpsHack = false; - - // Give the decoder the SPS again with the proper profile now - needsSpsReplay = true; - } + // log2_max_mv_length_horizontal and log2_max_mv_length_vertical are set to more + // conservative values by GFE 2.5.11. We'll let those values stand. } - else if (header.data[header.offset+4] == 0x40) { - numVpsIn++; + else { + // Devices that didn't/couldn't get bitstream restrictions before GFE 2.5.11 + // will continue to not receive them now + sps.vuiParams.bitstreamRestriction = null; } - else if (header.data[header.offset+4] == 0x42) { - numSpsIn++; + + // If we need to hack this SPS to say we're baseline, do so now + if (needsBaselineSpsHack) { + LimeLog.info("Hacking SPS to baseline"); + sps.profile_idc = 66; + savedSps = sps; } - else if (header.data[header.offset+4] == 0x44) { - numPpsIn++; + + // Patch the SPS constraint flags + doProfileSpecificSpsPatching(sps); + + // Write the annex B header + buf.put(frameData, 0, 5); + + // The H264Utils.writeSPS function safely handles + // Annex B NALUs (including NALUs with escape sequences) + ByteBuffer escapedNalu = H264Utils.writeSPS(sps, frameData.length); + buf.put(escapedNalu); + + queueInputBuffer(inputBufferIndex, + 0, buf.position(), + timestampUs, codecFlags); + + return MoonBridge.DR_OK; + + // H264 PPS + } else if (frameData[4] == 0x68) { + numPpsIn++; + codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; + + + if (needsBaselineSpsHack) { + LimeLog.info("Saw PPS; disabling SPS hack"); + needsBaselineSpsHack = false; + + // Give the decoder the SPS again with the proper profile now + needsSpsReplay = true; } } + else if (frameData[4] == 0x40) { + numVpsIn++; + codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; + } + else if (frameData[4] == 0x42) { + numSpsIn++; + codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; + } + else if (frameData[4] == 0x44) { + numPpsIn++; + codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; + } // Copy data from our buffer list into the input buffer - for (ByteBufferDescriptor desc = decodeUnit.getBufferHead(); - desc != null; desc = desc.nextDescriptor) { - buf.put(desc.data, desc.offset, desc.length); - } + buf.put(frameData); queueInputBuffer(inputBufferIndex, - 0, decodeUnit.getDataLength(), + 0, frameData.length, timestampUs, codecFlags); - depacketizer.freeDecodeUnit(decodeUnit); - if (needsSpsReplay) { replaySps(); } + + return MoonBridge.DR_OK; } private void replaySps() { @@ -774,24 +615,12 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { public int getCapabilities() { int caps = 0; - caps |= adaptivePlayback ? - VideoDecoderRenderer.CAPABILITY_ADAPTIVE_RESOLUTION : 0; - caps |= directSubmit ? - VideoDecoderRenderer.CAPABILITY_DIRECT_SUBMIT : 0; + MoonBridge.CAPABILITY_DIRECT_SUBMIT : 0; return caps; } - @Override - public int getAverageDecoderLatency() { - if (totalFrames == 0) { - return 0; - } - return (int)(decoderTimeMs / totalFrames); - } - - @Override public int getAverageEndToEndLatency() { if (totalFrames == 0) { return 0; @@ -799,33 +628,11 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { return (int)(totalTimeMs / totalFrames); } - private void notifyDuReceived(DecodeUnit du) { - long currentTime = MediaCodecHelper.getMonotonicMillis(); - long delta = currentTime-du.getReceiveTimestamp(); - if (delta >= 0 && delta < 1000) { - totalTimeMs += currentTime-du.getReceiveTimestamp(); - totalFrames++; - } - } - - @Override - public void directSubmitDecodeUnit(DecodeUnit du) { - int inputIndex = -1; - - notifyDuReceived(du); - - while (!Thread.currentThread().isInterrupted()) { - try { - inputIndex = dequeueInputBuffer(true, true); - break; - } catch (Exception e) { - handleDecoderException(e, null, 0); - } - } - - if (inputIndex >= 0) { - submitDecodeUnit(du, inputIndex); + public int getAverageDecoderLatency() { + if (totalFrames == 0) { + return 0; } + return (int)(decoderTimeMs / totalFrames); } public class RendererException extends RuntimeException { @@ -855,7 +662,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer { str += "AVC Decoder: "+((renderer.avcDecoder != null) ? renderer.avcDecoder.getName():"(none)")+"\n"; str += "HEVC Decoder: "+((renderer.hevcDecoder != null) ? renderer.hevcDecoder.getName():"(none)")+"\n"; str += "Initial video dimensions: "+renderer.initialWidth+"x"+renderer.initialHeight+"\n"; - str += "In stats: "+renderer.numVpsIn+", "+renderer.numSpsIn+", "+renderer.numPpsIn+", "+renderer.numIframeIn+"\n"; + str += "In stats: "+renderer.numVpsIn+", "+renderer.numSpsIn+", "+renderer.numPpsIn+"\n"; str += "Total frames: "+renderer.totalFrames+"\n"; str += "Average end-to-end client latency: "+getAverageEndToEndLatency()+"ms\n"; str += "Average hardware decoder latency: "+getAverageDecoderLatency()+"ms\n"; diff --git a/app/src/main/java/com/limelight/utils/Vector2d.java b/app/src/main/java/com/limelight/utils/Vector2d.java new file mode 100644 index 00000000..d01b78ba --- /dev/null +++ b/app/src/main/java/com/limelight/utils/Vector2d.java @@ -0,0 +1,47 @@ +package com.limelight.utils; + +public class Vector2d { + private float x; + private float y; + private double magnitude; + + public static final Vector2d ZERO = new Vector2d(); + + public Vector2d() { + initialize(0, 0); + } + + public void initialize(float x, float y) { + this.x = x; + this.y = y; + this.magnitude = Math.sqrt(Math.pow(x, 2) + Math.pow(y, 2)); + } + + public double getMagnitude() { + return magnitude; + } + + public void getNormalized(Vector2d vector) { + vector.initialize((float)(x / magnitude), (float)(y / magnitude)); + } + + public void scalarMultiply(double factor) { + initialize((float)(x * factor), (float)(y * factor)); + } + + public void setX(float x) { + initialize(x, this.y); + } + + public void setY(float y) { + initialize(this.x, y); + } + + public float getX() { + return x; + } + + public float getY() { + return y; + } +} diff --git a/moonlight-common b/moonlight-common index 16818d41..f5b8743d 160000 --- a/moonlight-common +++ b/moonlight-common @@ -1 +1 @@ -Subproject commit 16818d41068fdd9005f2a9f159c72054766c008c +Subproject commit f5b8743d693af4884e65f45243a8507fc598f40d