From 82619063ee1bd16525fac181971c5214ee70ac2c Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sun, 29 May 2022 21:58:28 -0500 Subject: [PATCH] Plumb frame type information into the decoder --- .../binding/video/MediaCodecDecoderRenderer.java | 6 +++++- .../nvstream/av/video/VideoDecoderRenderer.java | 2 +- .../main/java/com/limelight/nvstream/jni/MoonBridge.java | 9 ++++++--- app/src/main/jni/moonlight-core/callbacks.c | 8 ++++---- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index 285565fa..60cc01bd 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -714,7 +714,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C @SuppressWarnings("deprecation") @Override public int submitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int decodeUnitType, - int frameNumber, long receiveTimeMs, long enqueueTimeMs) { + int frameNumber, int frameType, long receiveTimeMs, long enqueueTimeMs) { if (stopping) { // Don't bother if we're stopping return MoonBridge.DR_OK; @@ -997,6 +997,10 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C submitCsdNextCall = false; } + if (frameType == MoonBridge.FRAME_TYPE_IDR) { + codecFlags |= MediaCodec.BUFFER_FLAG_SYNC_FRAME; + } + numFramesIn++; } diff --git a/app/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java b/app/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java index 5de9fa7e..e0484d91 100644 --- a/app/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java +++ b/app/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java @@ -10,7 +10,7 @@ public abstract class VideoDecoderRenderer { // This is called once for each frame-start NALU. This means it will be called several times // for an IDR frame which contains several parameter sets and the I-frame data. public abstract int submitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int decodeUnitType, - int frameNumber, long receiveTimeMs, long enqueueTimeMs); + int frameNumber, int frameType, long receiveTimeMs, long enqueueTimeMs); public abstract void cleanup(); diff --git a/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java b/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java index 739670b3..d50bc186 100644 --- a/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java +++ b/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java @@ -27,6 +27,9 @@ public class MoonBridge { public static final int BUFFER_TYPE_PPS = 2; public static final int BUFFER_TYPE_VPS = 3; + public static final int FRAME_TYPE_PFRAME = 0; + public static final int FRAME_TYPE_IDR = 1; + public static final int CAPABILITY_DIRECT_SUBMIT = 1; public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC = 2; public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC = 4; @@ -153,12 +156,12 @@ public class MoonBridge { } } - public static int bridgeDrSubmitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, - int decodeUnitType, int frameNumber, + public static int bridgeDrSubmitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int decodeUnitType, + int frameNumber, int frameType, long receiveTimeMs, long enqueueTimeMs) { if (videoRenderer != null) { return videoRenderer.submitDecodeUnit(decodeUnitData, decodeUnitLength, - decodeUnitType, frameNumber, receiveTimeMs, enqueueTimeMs); + decodeUnitType, frameNumber, frameType, receiveTimeMs, enqueueTimeMs); } else { return DR_OK; diff --git a/app/src/main/jni/moonlight-core/callbacks.c b/app/src/main/jni/moonlight-core/callbacks.c index a781e205..5e85bc2f 100644 --- a/app/src/main/jni/moonlight-core/callbacks.c +++ b/app/src/main/jni/moonlight-core/callbacks.c @@ -80,7 +80,7 @@ Java_com_limelight_nvstream_jni_MoonBridge_init(JNIEnv *env, jclass clazz) { BridgeDrStartMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrStart", "()V"); BridgeDrStopMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrStop", "()V"); BridgeDrCleanupMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrCleanup", "()V"); - BridgeDrSubmitDecodeUnitMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrSubmitDecodeUnit", "([BIIIJJ)I"); + BridgeDrSubmitDecodeUnitMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeDrSubmitDecodeUnit", "([BIIIIJJ)I"); BridgeArInitMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeArInit", "(III)I"); BridgeArStartMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeArStart", "()V"); BridgeArStopMethod = (*env)->GetStaticMethodID(env, clazz, "bridgeArStop", "()V"); @@ -159,8 +159,8 @@ int BridgeDrSubmitDecodeUnit(PDECODE_UNIT decodeUnit) { ret = (*env)->CallStaticIntMethod(env, GlobalBridgeClass, BridgeDrSubmitDecodeUnitMethod, DecodedFrameBuffer, currentEntry->length, currentEntry->bufferType, - decodeUnit->frameNumber, (jlong)decodeUnit->receiveTimeMs, - (jlong)decodeUnit->enqueueTimeMs); + decodeUnit->frameNumber, decodeUnit->frameType, + (jlong)decodeUnit->receiveTimeMs, (jlong)decodeUnit->enqueueTimeMs); if ((*env)->ExceptionCheck(env)) { // We will crash here (*JVM)->DetachCurrentThread(JVM); @@ -180,7 +180,7 @@ int BridgeDrSubmitDecodeUnit(PDECODE_UNIT decodeUnit) { ret = (*env)->CallStaticIntMethod(env, GlobalBridgeClass, BridgeDrSubmitDecodeUnitMethod, DecodedFrameBuffer, offset, BUFFER_TYPE_PICDATA, - decodeUnit->frameNumber, + decodeUnit->frameNumber, decodeUnit->frameType, (jlong)decodeUnit->receiveTimeMs, (jlong)decodeUnit->enqueueTimeMs); if ((*env)->ExceptionCheck(env)) { // We will crash here