moonlight-stream/moonlight-android
1
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-android.git synced 2025-07-19 02:53:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adapt to new-core reworking of moonlight-common (likely buggy)

Browse Source
This commit is contained in:
Cameron Gutman 2017-05-14 17:14:45 -07:00
parent 244fae07ab
commit 81d1e615bf
7 changed files with 249 additions and 432 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
app/src/main/java/com/limelight/Game.java
View File

@ -10,16 +10,15 @@ import com.limelight.binding.input.TouchContext;
import com.limelight.binding.input.driver.UsbDriverService;
import com.limelight.binding.input.evdev.EvdevListener;
import com.limelight.binding.input.virtual_controller.VirtualController;
import com.limelight.binding.video.EnhancedDecoderRenderer;
import com.limelight.binding.video.MediaCodecDecoderRenderer;
import com.limelight.binding.video.MediaCodecHelper;
import com.limelight.nvstream.NvConnection;
import com.limelight.nvstream.NvConnectionListener;
import com.limelight.nvstream.StreamConfiguration;
import com.limelight.nvstream.av.video.VideoDecoderRenderer;
import com.limelight.nvstream.http.NvApp;
import com.limelight.nvstream.input.KeyboardPacket;
import com.limelight.nvstream.input.MouseButtonPacket;
import com.limelight.nvstream.jni.MoonBridge;
import com.limelight.preferences.PreferenceConfiguration;
import com.limelight.ui.GameGestures;
import com.limelight.ui.StreamView;
@ -35,7 +34,6 @@ import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.ServiceConnection;
import android.content.res.Configuration;
import android.graphics.Point;
import android.hardware.input.InputManager;
import android.media.AudioManager;
@ -61,8 +59,6 @@ import android.widget.FrameLayout;
import android.view.inputmethod.InputMethodManager;
import android.widget.Toast;
import java.util.Locale;
public class Game extends Activity implements SurfaceHolder.Callback,
OnGenericMotionListener, OnTouchListener, NvConnectionListener, EvdevListener,
@ -83,7 +79,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
private ControllerHandler controllerHandler;
private VirtualController virtualController;
private KeyboardTranslator keybTranslator;
private PreferenceConfiguration prefConfig;
@ -101,12 +96,10 @@ public class Game extends Activity implements SurfaceHolder.Callback,
private ShortcutHelper shortcutHelper;
private EnhancedDecoderRenderer decoderRenderer;
private MediaCodecDecoderRenderer decoderRenderer;
private WifiManager.WifiLock wifiLock;
private int drFlags = 0;
private boolean connectedToUsbDriverService = false;
private ServiceConnection usbDriverServiceConnection = new ServiceConnection() {
@Override
@ -173,10 +166,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
// Read the stream preferences
prefConfig = PreferenceConfiguration.readPreferences(this);
if (prefConfig.stretchVideo) {
drFlags |= VideoDecoderRenderer.FLAG_FILL_SCREEN;
}
// Listen for events on the game surface
streamView = (StreamView) findViewById(R.id.surfaceView);
streamView.setOnGenericMotionListener(this);
@ -236,8 +225,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
.setApp(new NvApp(appName, appId))
.setBitrate(prefConfig.bitrate * 1000)
.setEnableSops(prefConfig.enableSops)
.enableAdaptiveResolution((decoderRenderer.getCapabilities() &
VideoDecoderRenderer.CAPABILITY_ADAPTIVE_RESOLUTION) != 0)
.enableLocalAudioPlayback(prefConfig.playHostAudio)
.setMaxPacketSize(remote ? 1024 : 1292)
.setRemote(remote)
@ -249,7 +236,6 @@ public class Game extends Activity implements SurfaceHolder.Callback,
// Initialize the connection
conn = new NvConnection(host, uniqueId, Game.this, config, PlatformBinding.getCryptoProvider(this));
keybTranslator = new KeyboardTranslator(conn);
controllerHandler = new ControllerHandler(this, conn, this, prefConfig.multiController, prefConfig.deadzonePercentage);
InputManager inputManager = (InputManager) getSystemService(Context.INPUT_SERVICE);
@ -438,7 +424,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
}
if (conn != null) {
VideoDecoderRenderer.VideoFormat videoFormat = conn.getActiveVideoFormat();
int videoFormat = conn.getActiveVideoFormat();
displayedFailureDialog = true;
stopConnection();
@ -457,11 +443,11 @@ public class Game extends Activity implements SurfaceHolder.Callback,
}
// Add the video codec to the post-stream toast
if (message != null && videoFormat != VideoDecoderRenderer.VideoFormat.Unknown) {
if (videoFormat == VideoDecoderRenderer.VideoFormat.H265) {
if (message != null) {
if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) {
message += " [H.265]";
}
else {
else if (videoFormat == MoonBridge.VIDEO_FORMAT_H264) {
message += " [H.264]";
}
}
@ -584,7 +570,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
if (!handled) {
// Try the keyboard handler
short translated = keybTranslator.translate(event.getKeyCode());
short translated = KeyboardTranslator.translate(event.getKeyCode());
if (translated == 0) {
return super.onKeyDown(keyCode, event);
}
@ -599,8 +585,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
return super.onKeyDown(keyCode, event);
}
keybTranslator.sendKeyDown(translated,
getModifierState(event));
conn.sendKeyboardInput(translated, KeyboardPacket.KEY_DOWN, getModifierState());
}
return true;
@ -623,7 +608,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
if (!handled) {
// Try the keyboard handler
short translated = keybTranslator.translate(event.getKeyCode());
short translated = KeyboardTranslator.translate(event.getKeyCode());
if (translated == 0) {
return super.onKeyUp(keyCode, event);
}
@ -637,8 +622,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
return super.onKeyUp(keyCode, event);
}
keybTranslator.sendKeyUp(translated,
getModifierState(event));
conn.sendKeyboardInput(translated, KeyboardPacket.KEY_UP, getModifierState(event));
}
return true;
@ -880,14 +864,14 @@ public class Game extends Activity implements SurfaceHolder.Callback,
}
@Override
public void stageStarting(Stage stage) {
public void stageStarting(String stage) {
if (spinner != null) {
spinner.setMessage(getResources().getString(R.string.conn_starting)+" "+stage.getName());
spinner.setMessage(getResources().getString(R.string.conn_starting)+" "+stage);
}
}
@Override
public void stageComplete(Stage stage) {
public void stageComplete(String stage) {
}
private void stopConnection() {
@ -904,7 +888,7 @@ public class Game extends Activity implements SurfaceHolder.Callback,
}
@Override
public void stageFailed(Stage stage) {
public void stageFailed(String stage, long errorCode) {
if (spinner != null) {
spinner.dismiss();
spinner = null;
@ -912,17 +896,19 @@ public class Game extends Activity implements SurfaceHolder.Callback,
if (!displayedFailureDialog) {
displayedFailureDialog = true;
LimeLog.severe(stage+" failed: "+errorCode);
stopConnection();
Dialog.displayDialog(this, getResources().getString(R.string.conn_error_title),
getResources().getString(R.string.conn_error_msg)+" "+stage.getName(), true);
getResources().getString(R.string.conn_error_msg)+" "+stage, true);
}
}
@Override
public void connectionTerminated(Exception e) {
public void connectionTerminated(long errorCode) {
if (!displayedFailureDialog) {
displayedFailureDialog = true;
e.printStackTrace();
LimeLog.severe("Connection terminated: "+errorCode);
stopConnection();
Dialog.displayDialog(this, getResources().getString(R.string.conn_terminated_title),
@ -984,8 +970,8 @@ public class Game extends Activity implements SurfaceHolder.Callback,
if (!connected && !connecting) {
connecting = true;
conn.start(PlatformBinding.getDeviceName(), holder, drFlags,
PlatformBinding.getAudioRenderer(), decoderRenderer);
decoderRenderer.setRenderTarget(holder);
conn.start(PlatformBinding.getAudioRenderer(), decoderRenderer);
}
}
@ -1044,17 +1030,17 @@ public class Game extends Activity implements SurfaceHolder.Callback,
@Override
public void keyboardEvent(boolean buttonDown, short keyCode) {
short keyMap = keybTranslator.translate(keyCode);
short keyMap = KeyboardTranslator.translate(keyCode);
if (keyMap != 0) {
if (handleSpecialKeys(keyMap, buttonDown)) {
return;
}
if (buttonDown) {
keybTranslator.sendKeyDown(keyMap, getModifierState());
conn.sendKeyboardInput(keyMap, KeyboardPacket.KEY_DOWN, getModifierState());
}
else {
keybTranslator.sendKeyUp(keyMap, getModifierState());
conn.sendKeyboardInput(keyMap, KeyboardPacket.KEY_UP, getModifierState());
}
}
}

47
app/src/main/java/com/limelight/binding/audio/AndroidAudioRenderer.java
View File

@ -6,34 +6,31 @@ import android.media.AudioTrack;
import com.limelight.LimeLog;
import com.limelight.nvstream.av.audio.AudioRenderer;
import com.limelight.nvstream.jni.MoonBridge;
public class AndroidAudioRenderer implements AudioRenderer {
private AudioTrack track;
@Override
public boolean streamInitialized(int channelCount, int channelMask, int samplesPerFrame, int sampleRate) {
public void setup(int audioConfiguration) {
int channelConfig;
int bufferSize;
int bytesPerFrame = (samplesPerFrame * 2);
int bytesPerFrame;
switch (channelCount)
switch (audioConfiguration)
{
case 1:
channelConfig = AudioFormat.CHANNEL_OUT_MONO;
break;
case 2:
channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
break;
case 4:
channelConfig = AudioFormat.CHANNEL_OUT_QUAD;
break;
case 6:
channelConfig = AudioFormat.CHANNEL_OUT_5POINT1;
break;
default:
LimeLog.severe("Decoder returned unhandled channel count");
return false;
case MoonBridge.AUDIO_CONFIGURATION_STEREO:
channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
bytesPerFrame = 2 * 240 * 2;
break;
case MoonBridge.AUDIO_CONFIGURATION_51_SURROUND:
channelConfig = AudioFormat.CHANNEL_OUT_5POINT1;
bytesPerFrame = 6 * 240 * 2;
break;
default:
LimeLog.severe("Decoder returned unhandled channel count");
return;
}
// We're not supposed to request less than the minimum
@ -46,7 +43,7 @@ public class AndroidAudioRenderer implements AudioRenderer {
bufferSize = bytesPerFrame * 2;
track = new AudioTrack(AudioManager.STREAM_MUSIC,
sampleRate,
48000,
channelConfig,
AudioFormat.ENCODING_PCM_16BIT,
bufferSize,
@ -61,7 +58,7 @@ public class AndroidAudioRenderer implements AudioRenderer {
} catch (Exception ignored) {}
// Now try the larger buffer size
bufferSize = Math.max(AudioTrack.getMinBufferSize(sampleRate,
bufferSize = Math.max(AudioTrack.getMinBufferSize(48000,
channelConfig,
AudioFormat.ENCODING_PCM_16BIT),
bytesPerFrame * 2);
@ -70,7 +67,7 @@ public class AndroidAudioRenderer implements AudioRenderer {
bufferSize = (((bufferSize + (bytesPerFrame - 1)) / bytesPerFrame) * bytesPerFrame);
track = new AudioTrack(AudioManager.STREAM_MUSIC,
sampleRate,
48000,
channelConfig,
AudioFormat.ENCODING_PCM_16BIT,
bufferSize,
@ -79,17 +76,15 @@ public class AndroidAudioRenderer implements AudioRenderer {
}
LimeLog.info("Audio track buffer size: "+bufferSize);
return true;
}
@Override
public void playDecodedAudio(byte[] audioData, int offset, int length) {
track.write(audioData, offset, length);
public void playDecodedAudio(byte[] audioData) {
track.write(audioData, 0, audioData.length);
}
@Override
public void streamClosing() {
public void cleanup() {
if (track != null) {
track.release();
}

14
app/src/main/java/com/limelight/binding/input/KeyboardTranslator.java
View File

@ -3,14 +3,13 @@ package com.limelight.binding.input;
import android.view.KeyEvent;
import com.limelight.nvstream.NvConnection;
import com.limelight.nvstream.input.KeycodeTranslator;
/**
* Class to translate a Android key code into the codes GFE is expecting
* @author Diego Waxemberg
* @author Cameron Gutman
*/
public class KeyboardTranslator extends KeycodeTranslator {
public class KeyboardTranslator {
/**
* GFE's prefix for every key code
@ -59,21 +58,12 @@ public class KeyboardTranslator extends KeycodeTranslator {
public static final int VK_QUOTE = 222;
public static final int VK_PAUSE = 19;
/**
* Constructs a new translator for the specified connection
* @param conn the connection to which the translated codes are sent
*/
public KeyboardTranslator(NvConnection conn) {
super(conn);
}
/**
* Translates the given keycode and returns the GFE keycode
* @param keycode the code to be translated
* @return a GFE keycode for the given keycode
*/
@Override
public short translate(int keycode) {
public static short translate(int keycode) {
int translated;
/* There seems to be no clean mapping between Android key codes

8
app/src/main/java/com/limelight/binding/video/EnhancedDecoderRenderer.java
View File

@ -1,8 +0,0 @@
package com.limelight.binding.video;
import com.limelight.nvstream.av.video.VideoDecoderRenderer;
public abstract class EnhancedDecoderRenderer extends VideoDecoderRenderer {
public abstract boolean isHevcSupported();
public abstract boolean isAvcSupported();
}

501
app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java
View File

@ -2,17 +2,14 @@ package com.limelight.binding.video;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.concurrent.locks.LockSupport;
import org.jcodec.codecs.h264.H264Utils;
import org.jcodec.codecs.h264.io.model.SeqParameterSet;
import org.jcodec.codecs.h264.io.model.VUIParameters;
import com.limelight.LimeLog;
import com.limelight.nvstream.av.ByteBufferDescriptor;
import com.limelight.nvstream.av.DecodeUnit;
import com.limelight.nvstream.av.video.VideoDecoderRenderer;
import com.limelight.nvstream.av.video.VideoDepacketizer;
import com.limelight.nvstream.jni.MoonBridge;
import com.limelight.preferences.PreferenceConfiguration;
import android.media.MediaCodec;
@ -23,7 +20,7 @@ import android.media.MediaCodec.CodecException;
import android.os.Build;
import android.view.SurfaceHolder;
public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
public class MediaCodecDecoderRenderer extends VideoDecoderRenderer {
private static final boolean USE_FRAME_RENDER_TIME = false;
@ -36,24 +33,23 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
private MediaCodec videoDecoder;
private Thread rendererThread;
private boolean needsSpsBitstreamFixup, isExynos4;
private VideoDepacketizer depacketizer;
private boolean adaptivePlayback, directSubmit;
private boolean constrainedHighProfile;
private int initialWidth, initialHeight;
private VideoFormat videoFormat;
private int videoFormat;
private Object renderTarget;
private boolean needsBaselineSpsHack;
private SeqParameterSet savedSps;
private long lastTimestampUs;
private long totalTimeMs;
private long decoderTimeMs;
private long totalTimeMs;
private int totalFrames;
private int numSpsIn;
private int numPpsIn;
private int numVpsIn;
private int numIframeIn;
private MediaCodecInfo findAvcDecoder() {
MediaCodecInfo decoder = MediaCodecHelper.findProbableSafeDecoder("video/avc", MediaCodecInfo.CodecProfileLevel.AVCProfileHigh);
@ -91,6 +87,10 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
return decoderInfo;
}
public void setRenderTarget(Object renderTarget) {
this.renderTarget = renderTarget;
}
public MediaCodecDecoderRenderer(int videoFormat) {
//dumpDecoders();
@ -125,18 +125,16 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
}
}
@Override
public boolean isHevcSupported() {
return hevcDecoder != null;
}
@Override
public boolean isAvcSupported() {
return avcDecoder != null;
}
@Override
public boolean setup(VideoDecoderRenderer.VideoFormat format, int width, int height, int redrawRate, Object renderTarget, int drFlags) {
public boolean setup(int format, int width, int height, int redrawRate) {
this.initialWidth = width;
this.initialHeight = height;
this.videoFormat = format;
@ -144,7 +142,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
String mimeType;
String selectedDecoderName;
if (videoFormat == VideoFormat.H264) {
if (videoFormat == MoonBridge.VIDEO_FORMAT_H264) {
mimeType = "video/avc";
selectedDecoderName = avcDecoder.getName();
@ -171,7 +169,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
LimeLog.info("Decoder "+selectedDecoderName+" is on Exynos 4");
}
}
else if (videoFormat == VideoFormat.H265) {
else if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) {
mimeType = "video/hevc";
selectedDecoderName = hevcDecoder.getName();
@ -229,6 +227,15 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
LimeLog.info("Using codec "+selectedDecoderName+" for hardware decoding "+mimeType);
// Start the decoder
videoDecoder.start();
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
legacyInputBuffers = videoDecoder.getInputBuffers();
}
startRendererThread();
return true;
}
@ -258,7 +265,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
}
}
private void startDirectSubmitRendererThread()
private void startRendererThread()
{
rendererThread = new Thread() {
@Override
@ -334,164 +341,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
return index;
}
private void startLegacyRendererThread()
{
rendererThread = new Thread() {
@Override
public void run() {
BufferInfo info = new BufferInfo();
DecodeUnit du = null;
int inputIndex = -1;
long lastDuDequeueTime = 0;
while (!isInterrupted())
{
// In order to get as much data to the decoder as early as possible,
// try to submit up to 5 decode units at once without blocking.
if (inputIndex == -1 && du == null) {
try {
for (int i = 0; i < 5; i++) {
inputIndex = dequeueInputBuffer(false, false);
du = depacketizer.pollNextDecodeUnit();
if (du != null) {
lastDuDequeueTime = MediaCodecHelper.getMonotonicMillis();
notifyDuReceived(du);
}
// Stop if we can't get a DU or input buffer
if (du == null || inputIndex == -1) {
break;
}
submitDecodeUnit(du, inputIndex);
du = null;
inputIndex = -1;
}
} catch (Exception e) {
inputIndex = -1;
handleDecoderException(e, null, 0);
}
}
// Grab an input buffer if we don't have one already.
// This way we can have one ready hopefully by the time
// the depacketizer is done with this frame. It's important
// that this can timeout because it's possible that we could exhaust
// the decoder's input buffers and deadlocks because aren't pulling
// frames out of the other end.
if (inputIndex == -1) {
try {
// If we've got a DU waiting to be given to the decoder,
// wait a full 3 ms for an input buffer. Otherwise
// just see if we can get one immediately.
inputIndex = dequeueInputBuffer(du != null, false);
} catch (Exception e) {
inputIndex = -1;
handleDecoderException(e, null, 0);
}
}
// Grab a decode unit if we don't have one already
if (du == null) {
du = depacketizer.pollNextDecodeUnit();
if (du != null) {
lastDuDequeueTime = MediaCodecHelper.getMonotonicMillis();
notifyDuReceived(du);
}
}
// If we've got both a decode unit and an input buffer, we'll
// submit now. Otherwise, we wait until we have one.
if (du != null && inputIndex >= 0) {
long submissionTime = MediaCodecHelper.getMonotonicMillis();
if (submissionTime - lastDuDequeueTime >= 20) {
LimeLog.warning("Receiving an input buffer took too long: "+(submissionTime - lastDuDequeueTime)+" ms");
}
submitDecodeUnit(du, inputIndex);
// DU and input buffer have both been consumed
du = null;
inputIndex = -1;
}
// Try to output a frame
try {
int outIndex = videoDecoder.dequeueOutputBuffer(info, 0);
if (outIndex >= 0) {
long presentationTimeUs = info.presentationTimeUs;
int lastIndex = outIndex;
// Get the last output buffer in the queue
while ((outIndex = videoDecoder.dequeueOutputBuffer(info, 0)) >= 0) {
videoDecoder.releaseOutputBuffer(lastIndex, false);
lastIndex = outIndex;
presentationTimeUs = info.presentationTimeUs;
}
// Render the last buffer
videoDecoder.releaseOutputBuffer(lastIndex, true);
// Add delta time to the totals (excluding probable outliers)
long delta = MediaCodecHelper.getMonotonicMillis()-(presentationTimeUs/1000);
if (delta >= 0 && delta < 1000) {
decoderTimeMs += delta;
if (!USE_FRAME_RENDER_TIME) {
totalTimeMs += delta;
}
}
} else {
switch (outIndex) {
case MediaCodec.INFO_TRY_AGAIN_LATER:
// Getting an input buffer may already block
// so don't park if we still need to do that
if (inputIndex >= 0) {
LockSupport.parkNanos(1);
}
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
LimeLog.info("Output format changed");
LimeLog.info("New output Format: " + videoDecoder.getOutputFormat());
break;
default:
break;
}
}
} catch (Exception e) {
handleDecoderException(e, null, 0);
}
}
}
};
rendererThread.setName("Video - Renderer (MediaCodec)");
rendererThread.setPriority(Thread.MAX_PRIORITY);
rendererThread.start();
}
@SuppressWarnings("deprecation")
@Override
public boolean start(VideoDepacketizer depacketizer) {
this.depacketizer = depacketizer;
// Start the decoder
videoDecoder.start();
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
legacyInputBuffers = videoDecoder.getInputBuffers();
}
if (directSubmit) {
startDirectSubmitRendererThread();
}
else {
startLegacyRendererThread();
}
return true;
}
@Override
// This method is used by the hack in Game, not called by the streaming core.
public void stop() {
if (rendererThread != null) {
// Halt the rendering thread
@ -500,13 +350,12 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
rendererThread.join();
} catch (InterruptedException ignored) { }
}
// We could stop the decoder here, but it seems to cause some problems
// so we'll just let release take care of it.
}
@Override
public void release() {
public void cleanup() {
stop();
if (videoDecoder != null) {
videoDecoder.release();
}
@ -570,7 +419,9 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
}
@SuppressWarnings("deprecation")
private void submitDecodeUnit(DecodeUnit decodeUnit, int inputBufferIndex) {
public int submitDecodeUnit(byte[] frameData) {
totalFrames++;
long timestampUs = System.nanoTime() / 1000;
if (timestampUs <= lastTimestampUs) {
// We can't submit multiple buffers with the same timestamp
@ -579,165 +430,155 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
}
lastTimestampUs = timestampUs;
int inputBufferIndex = dequeueInputBuffer(true, false);
ByteBuffer buf = getEmptyInputBuffer(inputBufferIndex);
int codecFlags = 0;
int decodeUnitFlags = decodeUnit.getFlags();
if ((decodeUnitFlags & DecodeUnit.DU_FLAG_CODEC_CONFIG) != 0) {
codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
}
if ((decodeUnitFlags & DecodeUnit.DU_FLAG_SYNC_FRAME) != 0) {
codecFlags |= MediaCodec.BUFFER_FLAG_SYNC_FRAME;
numIframeIn++;
}
boolean needsSpsReplay = false;
if ((decodeUnitFlags & DecodeUnit.DU_FLAG_CODEC_CONFIG) != 0) {
ByteBufferDescriptor header = decodeUnit.getBufferHead();
// H264 SPS
if (frameData[4] == 0x67) {
numSpsIn++;
codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
// H264 SPS
if (header.data[header.offset+4] == 0x67) {
numSpsIn++;
ByteBuffer spsBuf = ByteBuffer.wrap(frameData);
ByteBuffer spsBuf = ByteBuffer.wrap(header.data);
// Skip to the start of the NALU data
spsBuf.position(5);
// Skip to the start of the NALU data
spsBuf.position(header.offset+5);
// The H264Utils.readSPS function safely handles
// Annex B NALUs (including NALUs with escape sequences)
SeqParameterSet sps = H264Utils.readSPS(spsBuf);
// The H264Utils.readSPS function safely handles
// Annex B NALUs (including NALUs with escape sequences)
SeqParameterSet sps = H264Utils.readSPS(spsBuf);
// Some decoders rely on H264 level to decide how many buffers are needed
// Since we only need one frame buffered, we'll set the level as low as we can
// for known resolution combinations
if (initialWidth == 1280 && initialHeight == 720) {
// Max 5 buffered frames at 1280x720x60
LimeLog.info("Patching level_idc to 32");
sps.level_idc = 32;
}
else if (initialWidth == 1920 && initialHeight == 1080) {
// Max 4 buffered frames at 1920x1080x64
LimeLog.info("Patching level_idc to 42");
sps.level_idc = 42;
}
else {
// Leave the profile alone (currently 5.0)
}
// Some decoders rely on H264 level to decide how many buffers are needed
// Since we only need one frame buffered, we'll set the level as low as we can
// for known resolution combinations
if (initialWidth == 1280 && initialHeight == 720) {
// Max 5 buffered frames at 1280x720x60
LimeLog.info("Patching level_idc to 32");
sps.level_idc = 32;
}
else if (initialWidth == 1920 && initialHeight == 1080) {
// Max 4 buffered frames at 1920x1080x64
LimeLog.info("Patching level_idc to 42");
sps.level_idc = 42;
// TI OMAP4 requires a reference frame count of 1 to decode successfully. Exynos 4
// also requires this fixup.
//
// I'm doing this fixup for all devices because I haven't seen any devices that
// this causes issues for. At worst, it seems to do nothing and at best it fixes
// issues with video lag, hangs, and crashes.
LimeLog.info("Patching num_ref_frames in SPS");
sps.num_ref_frames = 1;
// GFE 2.5.11 changed the SPS to add additional extensions
// Some devices don't like these so we remove them here.
sps.vuiParams.video_signal_type_present_flag = false;
sps.vuiParams.colour_description_present_flag = false;
sps.vuiParams.chroma_loc_info_present_flag = false;
if (needsSpsBitstreamFixup || isExynos4) {
// The SPS that comes in the current H264 bytestream doesn't set bitstream_restriction_flag
// or max_dec_frame_buffering which increases decoding latency on Tegra.
// GFE 2.5.11 started sending bitstream restrictions
if (sps.vuiParams.bitstreamRestriction == null) {
LimeLog.info("Adding bitstream restrictions");
sps.vuiParams.bitstreamRestriction = new VUIParameters.BitstreamRestriction();
sps.vuiParams.bitstreamRestriction.motion_vectors_over_pic_boundaries_flag = true;
sps.vuiParams.bitstreamRestriction.log2_max_mv_length_horizontal = 16;
sps.vuiParams.bitstreamRestriction.log2_max_mv_length_vertical = 16;
sps.vuiParams.bitstreamRestriction.num_reorder_frames = 0;
}
else {
// Leave the profile alone (currently 5.0)
LimeLog.info("Patching bitstream restrictions");
}
// TI OMAP4 requires a reference frame count of 1 to decode successfully. Exynos 4
// also requires this fixup.
//
// I'm doing this fixup for all devices because I haven't seen any devices that
// this causes issues for. At worst, it seems to do nothing and at best it fixes
// issues with video lag, hangs, and crashes.
LimeLog.info("Patching num_ref_frames in SPS");
sps.num_ref_frames = 1;
// Some devices throw errors if max_dec_frame_buffering < num_ref_frames
sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering = sps.num_ref_frames;
// GFE 2.5.11 changed the SPS to add additional extensions
// Some devices don't like these so we remove them here.
sps.vuiParams.video_signal_type_present_flag = false;
sps.vuiParams.colour_description_present_flag = false;
sps.vuiParams.chroma_loc_info_present_flag = false;
// These values are the defaults for the fields, but they are more aggressive
// than what GFE sends in 2.5.11, but it doesn't seem to cause picture problems.
sps.vuiParams.bitstreamRestriction.max_bytes_per_pic_denom = 2;
sps.vuiParams.bitstreamRestriction.max_bits_per_mb_denom = 1;
if (needsSpsBitstreamFixup || isExynos4) {
// The SPS that comes in the current H264 bytestream doesn't set bitstream_restriction_flag
// or max_dec_frame_buffering which increases decoding latency on Tegra.
// GFE 2.5.11 started sending bitstream restrictions
if (sps.vuiParams.bitstreamRestriction == null) {
LimeLog.info("Adding bitstream restrictions");
sps.vuiParams.bitstreamRestriction = new VUIParameters.BitstreamRestriction();
sps.vuiParams.bitstreamRestriction.motion_vectors_over_pic_boundaries_flag = true;
sps.vuiParams.bitstreamRestriction.log2_max_mv_length_horizontal = 16;
sps.vuiParams.bitstreamRestriction.log2_max_mv_length_vertical = 16;
sps.vuiParams.bitstreamRestriction.num_reorder_frames = 0;
}
else {
LimeLog.info("Patching bitstream restrictions");
}
// Some devices throw errors if max_dec_frame_buffering < num_ref_frames
sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering = sps.num_ref_frames;
// These values are the defaults for the fields, but they are more aggressive
// than what GFE sends in 2.5.11, but it doesn't seem to cause picture problems.
sps.vuiParams.bitstreamRestriction.max_bytes_per_pic_denom = 2;
sps.vuiParams.bitstreamRestriction.max_bits_per_mb_denom = 1;
// log2_max_mv_length_horizontal and log2_max_mv_length_vertical are set to more
// conservative values by GFE 2.5.11. We'll let those values stand.
}
else {
// Devices that didn't/couldn't get bitstream restrictions before GFE 2.5.11
// will continue to not receive them now
sps.vuiParams.bitstreamRestriction = null;
}
// If we need to hack this SPS to say we're baseline, do so now
if (needsBaselineSpsHack) {
LimeLog.info("Hacking SPS to baseline");
sps.profile_idc = 66;
savedSps = sps;
}
// Patch the SPS constraint flags
doProfileSpecificSpsPatching(sps);
// Write the annex B header
buf.put(header.data, header.offset, 5);
// The H264Utils.writeSPS function safely handles
// Annex B NALUs (including NALUs with escape sequences)
ByteBuffer escapedNalu = H264Utils.writeSPS(sps, header.length);
buf.put(escapedNalu);
queueInputBuffer(inputBufferIndex,
0, buf.position(),
timestampUs, codecFlags);
depacketizer.freeDecodeUnit(decodeUnit);
return;
// H264 PPS
} else if (header.data[header.offset+4] == 0x68) {
numPpsIn++;
if (needsBaselineSpsHack) {
LimeLog.info("Saw PPS; disabling SPS hack");
needsBaselineSpsHack = false;
// Give the decoder the SPS again with the proper profile now
needsSpsReplay = true;
}
// log2_max_mv_length_horizontal and log2_max_mv_length_vertical are set to more
// conservative values by GFE 2.5.11. We'll let those values stand.
}
else if (header.data[header.offset+4] == 0x40) {
numVpsIn++;
else {
// Devices that didn't/couldn't get bitstream restrictions before GFE 2.5.11
// will continue to not receive them now
sps.vuiParams.bitstreamRestriction = null;
}
else if (header.data[header.offset+4] == 0x42) {
numSpsIn++;
// If we need to hack this SPS to say we're baseline, do so now
if (needsBaselineSpsHack) {
LimeLog.info("Hacking SPS to baseline");
sps.profile_idc = 66;
savedSps = sps;
}
else if (header.data[header.offset+4] == 0x44) {
numPpsIn++;
// Patch the SPS constraint flags
doProfileSpecificSpsPatching(sps);
// Write the annex B header
buf.put(frameData, 0, 5);
// The H264Utils.writeSPS function safely handles
// Annex B NALUs (including NALUs with escape sequences)
ByteBuffer escapedNalu = H264Utils.writeSPS(sps, frameData.length);
buf.put(escapedNalu);
queueInputBuffer(inputBufferIndex,
0, buf.position(),
timestampUs, codecFlags);
return MoonBridge.DR_OK;
// H264 PPS
} else if (frameData[4] == 0x68) {
numPpsIn++;
codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
if (needsBaselineSpsHack) {
LimeLog.info("Saw PPS; disabling SPS hack");
needsBaselineSpsHack = false;
// Give the decoder the SPS again with the proper profile now
needsSpsReplay = true;
}
}
else if (frameData[4] == 0x40) {
numVpsIn++;
codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
}
else if (frameData[4] == 0x42) {
numSpsIn++;
codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
}
else if (frameData[4] == 0x44) {
numPpsIn++;
codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG;
}
// Copy data from our buffer list into the input buffer
for (ByteBufferDescriptor desc = decodeUnit.getBufferHead();
desc != null; desc = desc.nextDescriptor) {
buf.put(desc.data, desc.offset, desc.length);
}
buf.put(frameData);
queueInputBuffer(inputBufferIndex,
0, decodeUnit.getDataLength(),
0, frameData.length,
timestampUs, codecFlags);
depacketizer.freeDecodeUnit(decodeUnit);
if (needsSpsReplay) {
replaySps();
}
return MoonBridge.DR_OK;
}
private void replaySps() {
@ -774,24 +615,12 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
public int getCapabilities() {
int caps = 0;
caps |= adaptivePlayback ?
VideoDecoderRenderer.CAPABILITY_ADAPTIVE_RESOLUTION : 0;
caps |= directSubmit ?
VideoDecoderRenderer.CAPABILITY_DIRECT_SUBMIT : 0;
MoonBridge.CAPABILITY_DIRECT_SUBMIT : 0;
return caps;
}
@Override
public int getAverageDecoderLatency() {
if (totalFrames == 0) {
return 0;
}
return (int)(decoderTimeMs / totalFrames);
}
@Override
public int getAverageEndToEndLatency() {
if (totalFrames == 0) {
return 0;
@ -799,33 +628,11 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
return (int)(totalTimeMs / totalFrames);
}
private void notifyDuReceived(DecodeUnit du) {
long currentTime = MediaCodecHelper.getMonotonicMillis();
long delta = currentTime-du.getReceiveTimestamp();
if (delta >= 0 && delta < 1000) {
totalTimeMs += currentTime-du.getReceiveTimestamp();
totalFrames++;
}
}
@Override
public void directSubmitDecodeUnit(DecodeUnit du) {
int inputIndex = -1;
notifyDuReceived(du);
while (!Thread.currentThread().isInterrupted()) {
try {
inputIndex = dequeueInputBuffer(true, true);
break;
} catch (Exception e) {
handleDecoderException(e, null, 0);
}
}
if (inputIndex >= 0) {
submitDecodeUnit(du, inputIndex);
public int getAverageDecoderLatency() {
if (totalFrames == 0) {
return 0;
}
return (int)(decoderTimeMs / totalFrames);
}
public class RendererException extends RuntimeException {
@ -855,7 +662,7 @@ public class MediaCodecDecoderRenderer extends EnhancedDecoderRenderer {
str += "AVC Decoder: "+((renderer.avcDecoder != null) ? renderer.avcDecoder.getName():"(none)")+"\n";
str += "HEVC Decoder: "+((renderer.hevcDecoder != null) ? renderer.hevcDecoder.getName():"(none)")+"\n";
str += "Initial video dimensions: "+renderer.initialWidth+"x"+renderer.initialHeight+"\n";
str += "In stats: "+renderer.numVpsIn+", "+renderer.numSpsIn+", "+renderer.numPpsIn+", "+renderer.numIframeIn+"\n";
str += "In stats: "+renderer.numVpsIn+", "+renderer.numSpsIn+", "+renderer.numPpsIn+"\n";
str += "Total frames: "+renderer.totalFrames+"\n";
str += "Average end-to-end client latency: "+getAverageEndToEndLatency()+"ms\n";
str += "Average hardware decoder latency: "+getAverageDecoderLatency()+"ms\n";

47
app/src/main/java/com/limelight/utils/Vector2d.java Normal file
View File

@ -0,0 +1,47 @@
package com.limelight.utils;
public class Vector2d {
private float x;
private float y;
private double magnitude;
public static final Vector2d ZERO = new Vector2d();
public Vector2d() {
initialize(0, 0);
}
public void initialize(float x, float y) {
this.x = x;
this.y = y;
this.magnitude = Math.sqrt(Math.pow(x, 2) + Math.pow(y, 2));
}
public double getMagnitude() {
return magnitude;
}
public void getNormalized(Vector2d vector) {
vector.initialize((float)(x / magnitude), (float)(y / magnitude));
}
public void scalarMultiply(double factor) {
initialize((float)(x * factor), (float)(y * factor));
}
public void setX(float x) {
initialize(x, this.y);
}
public void setY(float y) {
initialize(this.x, y);
}
public float getX() {
return x;
}
public float getY() {
return y;
}
}

2
moonlight-common

@ -1 +1 @@
Subproject commit 16818d41068fdd9005f2a9f159c72054766c008c
Subproject commit f5b8743d693af4884e65f45243a8507fc598f40d