From 73e4970a43372e350ddbc6c509d238f9e1c9df22 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sun, 14 May 2017 23:06:41 -0700 Subject: [PATCH] JNI code complete --- .../com/limelight/nvstream/NvConnection.java | 10 +- .../nvstream/StreamConfiguration.java | 15 +- .../nvstream/av/audio/AudioRenderer.java | 2 - .../av/video/VideoDecoderRenderer.java | 4 - .../limelight/nvstream/jni/MoonBridge.java | 13 +- .../src/main/jni/moonlight-core/Android.mk | 3 + .../src/main/jni/moonlight-core/callbacks.c | 253 ++++++++++++++++++ .../src/main/jni/moonlight-core/simplejni.c | 51 ++++ 8 files changed, 338 insertions(+), 13 deletions(-) create mode 100644 moonlight-common/src/main/jni/moonlight-core/callbacks.c create mode 100644 moonlight-common/src/main/jni/moonlight-core/simplejni.c diff --git a/moonlight-common/src/main/java/com/limelight/nvstream/NvConnection.java b/moonlight-common/src/main/java/com/limelight/nvstream/NvConnection.java index ed8e467f..bbca256e 100644 --- a/moonlight-common/src/main/java/com/limelight/nvstream/NvConnection.java +++ b/moonlight-common/src/main/java/com/limelight/nvstream/NvConnection.java @@ -3,6 +3,7 @@ package com.limelight.nvstream; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; +import java.nio.ByteBuffer; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; @@ -223,8 +224,15 @@ public class NvConnection { return; } + ByteBuffer ib = ByteBuffer.allocate(16); + ib.putInt(context.riKeyId); + MoonBridge.startConnection(context.serverAddress.getHostAddress(), - context.serverAppVersion, context.serverGfeVersion); + context.serverAppVersion, context.serverGfeVersion, + context.streamConfig.getWidth(), context.streamConfig.getHeight(), + context.streamConfig.getRefreshRate(), context.streamConfig.getBitrate(), + context.streamConfig.getRemote(), context.streamConfig.getAudioConfiguration(), + context.streamConfig.getHevcSupported(), context.riKey.getEncoded(), ib.array()); } public void start(AudioRenderer audioRenderer, VideoDecoderRenderer videoDecoderRenderer, NvConnectionListener connectionListener) diff --git a/moonlight-common/src/main/java/com/limelight/nvstream/StreamConfiguration.java b/moonlight-common/src/main/java/com/limelight/nvstream/StreamConfiguration.java index f7fe4579..4f182ff5 100644 --- a/moonlight-common/src/main/java/com/limelight/nvstream/StreamConfiguration.java +++ b/moonlight-common/src/main/java/com/limelight/nvstream/StreamConfiguration.java @@ -1,13 +1,11 @@ package com.limelight.nvstream; import com.limelight.nvstream.http.NvApp; +import com.limelight.nvstream.jni.MoonBridge; public class StreamConfiguration { public static final int INVALID_APP_ID = 0; - public static final int AUDIO_CONFIGURATION_STEREO = 1; - public static final int AUDIO_CONFIGURATION_5_1 = 2; - private static final int CHANNEL_COUNT_STEREO = 2; private static final int CHANNEL_COUNT_5_1 = 6; @@ -25,6 +23,7 @@ public class StreamConfiguration { private boolean remote; private int audioChannelMask; private int audioChannelCount; + private int audioConfiguration; private boolean supportsHevc; public static class Builder { @@ -77,11 +76,11 @@ public class StreamConfiguration { } public StreamConfiguration.Builder setAudioConfiguration(int audioConfig) { - if (audioConfig == AUDIO_CONFIGURATION_STEREO) { + if (audioConfig == MoonBridge.AUDIO_CONFIGURATION_STEREO) { config.audioChannelCount = CHANNEL_COUNT_STEREO; config.audioChannelMask = CHANNEL_MASK_STEREO; } - else if (audioConfig == AUDIO_CONFIGURATION_5_1) { + else if (audioConfig == MoonBridge.AUDIO_CONFIGURATION_51_SURROUND) { config.audioChannelCount = CHANNEL_COUNT_5_1; config.audioChannelMask = CHANNEL_MASK_5_1; } @@ -89,6 +88,8 @@ public class StreamConfiguration { throw new IllegalArgumentException("Invalid audio configuration"); } + config.audioConfiguration = audioConfig; + return this; } @@ -164,6 +165,10 @@ public class StreamConfiguration { public int getAudioChannelMask() { return audioChannelMask; } + + public int getAudioConfiguration() { + return audioConfiguration; + } public boolean getHevcSupported() { return supportsHevc; diff --git a/moonlight-common/src/main/java/com/limelight/nvstream/av/audio/AudioRenderer.java b/moonlight-common/src/main/java/com/limelight/nvstream/av/audio/AudioRenderer.java index 76c56316..af5e8556 100644 --- a/moonlight-common/src/main/java/com/limelight/nvstream/av/audio/AudioRenderer.java +++ b/moonlight-common/src/main/java/com/limelight/nvstream/av/audio/AudioRenderer.java @@ -1,8 +1,6 @@ package com.limelight.nvstream.av.audio; public interface AudioRenderer { - int getCapabilities(); - void setup(int audioConfiguration); void playDecodedAudio(byte[] audioData); diff --git a/moonlight-common/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java b/moonlight-common/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java index 7e7bbe76..76ffab1d 100644 --- a/moonlight-common/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java +++ b/moonlight-common/src/main/java/com/limelight/nvstream/av/video/VideoDecoderRenderer.java @@ -1,10 +1,6 @@ package com.limelight.nvstream.av.video; public abstract class VideoDecoderRenderer { - public int getCapabilities() { - return 0; - } - public abstract boolean setup(int format, int width, int height, int redrawRate); public abstract int submitDecodeUnit(byte[] frameData); diff --git a/moonlight-common/src/main/java/com/limelight/nvstream/jni/MoonBridge.java b/moonlight-common/src/main/java/com/limelight/nvstream/jni/MoonBridge.java index 22064855..01a56865 100644 --- a/moonlight-common/src/main/java/com/limelight/nvstream/jni/MoonBridge.java +++ b/moonlight-common/src/main/java/com/limelight/nvstream/jni/MoonBridge.java @@ -21,6 +21,11 @@ public class MoonBridge { private static VideoDecoderRenderer videoRenderer; private static NvConnectionListener connectionListener; + static { + System.load("moonlight-core"); + init(); + } + public static int CAPABILITY_SLICES_PER_FRAME(byte slices) { return slices << 24; } @@ -118,7 +123,11 @@ public class MoonBridge { MoonBridge.connectionListener = null; } - public static native void startConnection(String address, String serverInfoAppVersion, String serverInfoGfeVersion); + public static native void startConnection(String address, String appVersion, String gfeVersion, + int width, int height, int fps, + int bitrate, boolean streamingRemotely, + int audioConfiguration, boolean supportsHevc, + byte[] riAesKey, byte[] riAesIv); public static native void stopConnection(); @@ -142,4 +151,6 @@ public class MoonBridge { public static native void sendMouseScroll(byte scrollClicks); public static native String getStageName(int stage); + + public static native void init(); } diff --git a/moonlight-common/src/main/jni/moonlight-core/Android.mk b/moonlight-common/src/main/jni/moonlight-core/Android.mk index beaa593e..6d593927 100644 --- a/moonlight-common/src/main/jni/moonlight-core/Android.mk +++ b/moonlight-common/src/main/jni/moonlight-core/Android.mk @@ -35,6 +35,9 @@ LOCAL_SRC_FILES := moonlight-common-c/src/AudioStream.c \ moonlight-common-c/enet/protocol.c \ moonlight-common-c/enet/unix.c \ moonlight-common-c/enet/win32.c \ + simplejni.c \ + callbacks.c \ + LOCAL_C_INCLUDES := $(LOCAL_PATH)/moonlight-common-c/enet/include \ $(LOCAL_PATH)/moonlight-common-c/reedsolomon \ diff --git a/moonlight-common/src/main/jni/moonlight-core/callbacks.c b/moonlight-common/src/main/jni/moonlight-core/callbacks.c new file mode 100644 index 00000000..789b1409 --- /dev/null +++ b/moonlight-common/src/main/jni/moonlight-core/callbacks.c @@ -0,0 +1,253 @@ +#include + +#include + +#include + +#include + +#define PCM_FRAME_SIZE 240 + +static OpusMSDecoder* Decoder; +static OPUS_MULTISTREAM_CONFIGURATION OpusConfig; + +static JavaVM *JVM; +static pthread_key_t JniEnvKey; +static jclass GlobalBridgeClass; +static jmethodID BridgeDrSetupMethod; +static jmethodID BridgeDrCleanupMethod; +static jmethodID BridgeDrSubmitDecodeUnitMethod; +static jmethodID BridgeArInitMethod; +static jmethodID BridgeArCleanupMethod; +static jmethodID BridgeArPlaySampleMethod; +static jmethodID BridgeClStageStartingMethod; +static jmethodID BridgeClStageCompleteMethod; +static jmethodID BridgeClStageFailedMethod; +static jmethodID BridgeClConnectionStartedMethod; +static jmethodID BridgeClConnectionTerminatedMethod; +static jmethodID BridgeClDisplayMessageMethod; +static jmethodID BridgeClDisplayTransientMessageMethod; + +static void DetachThread(void* context) { + (*JVM)->DetachCurrentThread(JVM); +} + +static JNIEnv* GetThreadEnv(void) { + JNIEnv* env; + + // Try the TLS to see if we already have a JNIEnv + env = pthread_getspecific(JniEnvKey); + if (env) + return env; + + // This is the thread's first JNI call, so attach now + (*JVM)->AttachCurrentThread(JVM, &env, NULL); + + // Write our JNIEnv to TLS, so we detach before dying + pthread_setspecific(JniEnvKey, env); + + return env; +} + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_init(JNIEnv *env, jobject class) { + GlobalBridgeClass = (*env)->NewGlobalRef(env, (*env)->FindClass(env, "com/limelight/nvstream/jni/MoonBridge")); + BridgeDrSetupMethod = (*env)->GetStaticMethodID(env, class, "bridgeDrSetup", "(IIII)V"); + BridgeDrCleanupMethod = (*env)->GetStaticMethodID(env, class, "bridgeDrCleanup", "()V"); + BridgeDrSubmitDecodeUnitMethod = (*env)->GetStaticMethodID(env, class, "bridgeDrSubmitDecodeUnit", "([B)I"); + BridgeArInitMethod = (*env)->GetStaticMethodID(env, class, "bridgeArInit", "(I)V"); + BridgeArCleanupMethod = (*env)->GetStaticMethodID(env, class, "bridgeArCleanup", "()V"); + BridgeArPlaySampleMethod = (*env)->GetStaticMethodID(env, class, "bridgeArPlaySample", "([B)V"); + BridgeClStageStartingMethod = (*env)->GetStaticMethodID(env, class, "bridgeClStageStarting", "(I)V"); + BridgeClStageCompleteMethod = (*env)->GetStaticMethodID(env, class, "bridgeClStageComplete", "(I)V"); + BridgeClStageFailedMethod = (*env)->GetStaticMethodID(env, class, "bridgeClStageFailed", "(IL)V"); + BridgeClConnectionStartedMethod = (*env)->GetStaticMethodID(env, class, "bridgeClConnectionStarted", "()V"); + BridgeClConnectionTerminatedMethod = (*env)->GetStaticMethodID(env, class, "bridgeClConnectionTerminated", "(L)V"); + BridgeClDisplayMessageMethod = (*env)->GetStaticMethodID(env, class, "bridgeClDisplayMessage", "(Ljava/lang/String;)V"); + BridgeClDisplayTransientMessageMethod = (*env)->GetStaticMethodID(env, class, "bridgeClDisplayTransientMessage", "(Ljava/lang/String;)V"); + + // Create a TLS slot for the JNIEnv + pthread_key_create(&JniEnvKey, DetachThread); +} + +static void BridgeDrSetup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeDrSetupMethod, videoFormat, width, height, redrawRate); +} + +static void BridgeDrCleanup(void) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeDrCleanupMethod); +} + +static int BridgeDrSubmitDecodeUnit(PDECODE_UNIT decodeUnit) { + JNIEnv* env = GetThreadEnv(); + jbyteArray dataRef = (*env)->NewByteArray(env, decodeUnit->fullLength); + jbyte* data = (*env)->GetByteArrayElements(env, dataRef, 0); + PLENTRY currentEntry; + int offset; + + currentEntry = decodeUnit->bufferList; + offset = 0; + while (currentEntry != NULL) { + memcpy(&data[offset], currentEntry->data, currentEntry->length); + offset += currentEntry->length; + + currentEntry = currentEntry->next; + } + + // We must release the array elements first to ensure the data is copied before the callback + (*env)->ReleaseByteArrayElements(env, dataRef, data, 0); + + int ret = (*env)->CallStaticIntMethod(env, GlobalBridgeClass, BridgeDrSubmitDecodeUnitMethod, dataRef); + (*env)->DeleteLocalRef(env, dataRef); + + return ret; +} + +static void BridgeArInit(int audioConfiguration, POPUS_MULTISTREAM_CONFIGURATION opusConfig) { + JNIEnv* env = GetThreadEnv(); + int err; + + memcpy(&OpusConfig, opusConfig, sizeof(*opusConfig)); + Decoder = opus_multistream_decoder_create(opusConfig->sampleRate, + opusConfig->channelCount, + opusConfig->streams, + opusConfig->coupledStreams, + opusConfig->mapping, + &err); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeArInitMethod, audioConfiguration); +} + +static void BridgeArCleanup() { + JNIEnv* env = GetThreadEnv(); + + opus_multistream_decoder_destroy(Decoder); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeArCleanupMethod); +} + +static void BridgeArDecodeAndPlaySample(char* sampleData, int sampleLength) { + JNIEnv* env = GetThreadEnv(); + jbyteArray decodedDataRef = (*env)->NewByteArray(env, OpusConfig.channelCount * 240); + jbyte* decodedData = (*env)->GetByteArrayElements(env, decodedDataRef, 0); + + int decodeLen = opus_multistream_decode(Decoder, + (const unsigned char*)sampleData, + sampleLength, + (opus_int16*)decodedData, + PCM_FRAME_SIZE, + 0); + if (decodeLen > 0) { + // We must release the array elements first to ensure the data is copied before the callback + (*env)->ReleaseByteArrayElements(env, decodedDataRef, decodedData, 0); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeArPlaySampleMethod, decodedDataRef); + } + else { + // We can abort here to avoid the copy back since no data was modified + (*env)->ReleaseByteArrayElements(env, decodedDataRef, decodedData, JNI_ABORT); + } + + (*env)->DeleteLocalRef(env, decodedDataRef); +} + +static void BridgeClStageStarting(int stage) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeClStageStartingMethod, stage); +} + +static void BridgeClStageComplete(int stage) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeClStageCompleteMethod, stage); +} + +static void BridgeClStageFailed(int stage, long errorCode) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeClStageFailedMethod, stage, errorCode); +} + +static void BridgeClConnectionStarted(void) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeClConnectionStartedMethod, NULL); +} + +static void BridgeClConnectionTerminated(long errorCode) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeClConnectionTerminatedMethod, errorCode); +} + +static void BridgeClDisplayMessage(const char* message) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeClDisplayMessageMethod, (*env)->NewStringUTF(env, message)); +} + +static void BridgeClDisplayTransientMessage(const char* message) { + JNIEnv* env = GetThreadEnv(); + + (*env)->CallStaticVoidMethod(env, GlobalBridgeClass, BridgeClDisplayTransientMessageMethod, (*env)->NewStringUTF(env, message)); +} + +static DECODER_RENDERER_CALLBACKS BridgeVideoRendererCallbacks = { + .setup = BridgeDrSetup, + .cleanup = BridgeDrCleanup, + .submitDecodeUnit = BridgeDrSubmitDecodeUnit, +}; + +static AUDIO_RENDERER_CALLBACKS BridgeAudioRendererCallbacks = { + .init = BridgeArInit, + .cleanup = BridgeArCleanup, + .decodeAndPlaySample = BridgeArDecodeAndPlaySample, +}; + +static CONNECTION_LISTENER_CALLBACKS BridgeConnListenerCallbacks = { + .stageStarting = BridgeClStageStarting, + .stageComplete = BridgeClStageComplete, + .stageFailed = BridgeClStageFailed, + .connectionStarted = BridgeClConnectionStarted, + .connectionTerminated = BridgeClConnectionTerminated, + .displayMessage = BridgeClDisplayMessage, + .displayTransientMessage = BridgeClDisplayTransientMessage, +}; + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_startConnection(JNIEnv *env, jobject class, + jstring address, jstring appVersion, jstring gfeVersion, + jint width, jint height, jint fps, + jint bitrate, jboolean streamingRemotely, + jint audioConfiguration, jboolean supportsHevc, + jbyteArray riAesKey, jbyteArray riAesIv) { + SERVER_INFORMATION serverInfo = { + .address = address, + .serverInfoAppVersion = appVersion, + .serverInfoGfeVersion = gfeVersion, + }; + STREAM_CONFIGURATION streamConfig = { + .width = width, + .height = height, + .fps = fps, + .bitrate = bitrate, + .streamingRemotely = streamingRemotely, + .audioConfiguration = audioConfiguration, + .supportsHevc = supportsHevc, + }; + + jbyte* riAesKeyBuf = (*env)->GetByteArrayElements(env, riAesKey, NULL); + memcpy(streamConfig.remoteInputAesKey, riAesKeyBuf, sizeof(streamConfig.remoteInputAesKey)); + (*env)->ReleaseByteArrayElements(env, riAesKey, riAesKeyBuf, JNI_ABORT); + + jbyte* riAesIvBuf = (*env)->GetByteArrayElements(env, riAesIv, NULL); + memcpy(streamConfig.remoteInputAesIv, riAesIvBuf, sizeof(streamConfig.remoteInputAesIv)); + (*env)->ReleaseByteArrayElements(env, riAesIv, riAesIvBuf, JNI_ABORT); + + LiStartConnection(&serverInfo, &streamConfig, &BridgeConnListenerCallbacks, &BridgeVideoRendererCallbacks, &BridgeAudioRendererCallbacks, NULL, 0); +} \ No newline at end of file diff --git a/moonlight-common/src/main/jni/moonlight-core/simplejni.c b/moonlight-common/src/main/jni/moonlight-core/simplejni.c new file mode 100644 index 00000000..02047698 --- /dev/null +++ b/moonlight-common/src/main/jni/moonlight-core/simplejni.c @@ -0,0 +1,51 @@ +#include + +#include + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_sendMouseMove(JNIEnv *env, jobject class, jshort deltaX, jshort deltaY) { + LiSendMouseMoveEvent(deltaX, deltaY); +} + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_sendMouseButton(JNIEnv *env, jobject class, jbyte buttonEvent, jbyte mouseButton) { + LiSendMouseButtonEvent(buttonEvent, mouseButton); +} + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_sendMultiControllerInput(JNIEnv *env, jobject class, jshort controllerNumber, + jshort activeGamepadMask, jshort buttonFlags, + jbyte leftTrigger, jbyte rightTrigger, + jshort leftStickX, jshort leftStickY, + jshort rightStickX, jshort rightStickY) { + LiSendMultiControllerEvent(controllerNumber, activeGamepadMask, buttonFlags, + leftTrigger, rightTrigger, leftStickX, leftStickY, rightStickX, rightStickY); +} + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_sendControllerInput(JNIEnv *env, jobject class, jshort buttonFlags, + jbyte leftTrigger, jbyte rightTrigger, + jshort leftStickX, jshort leftStickY, + jshort rightStickX, jshort rightStickY) { + LiSendControllerEvent(buttonFlags, leftTrigger, rightTrigger, leftStickX, leftStickY, rightStickX, rightStickY); +} + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_sendKeyboardInput(JNIEnv *env, jobject class, jshort keyCode, jbyte keyAction, jbyte modifiers) { + LiSendKeyboardEvent(keyCode, keyAction, modifiers); +} + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_sendMouseScroll(JNIEnv *env, jobject class, jbyte scrollClicks) { + LiSendScrollEvent(scrollClicks); +} + +JNIEXPORT void JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_stopConnection(JNIEnv *env, jobject class) { + LiStopConnection(); +} + +JNIEXPORT jstring JNICALL +Java_com_limelight_nvstream_jni_MoonBridge_getStageName(JNIEnv *env, jobject class, jint stage) { + return (*env)->NewStringUTF(env, LiGetStageName(stage)); +} \ No newline at end of file