package com.limelight; import com.limelight.binding.PlatformBinding; import com.limelight.binding.input.ControllerHandler; import com.limelight.binding.input.KeyboardTranslator; import com.limelight.binding.input.capture.InputCaptureManager; import com.limelight.binding.input.capture.InputCaptureProvider; import com.limelight.binding.input.TouchContext; import com.limelight.binding.input.driver.UsbDriverService; import com.limelight.binding.input.evdev.EvdevListener; import com.limelight.binding.input.virtual_controller.VirtualController; import com.limelight.binding.video.CrashListener; import com.limelight.binding.video.MediaCodecDecoderRenderer; import com.limelight.binding.video.MediaCodecHelper; import com.limelight.nvstream.NvConnection; import com.limelight.nvstream.NvConnectionListener; import com.limelight.nvstream.StreamConfiguration; import com.limelight.nvstream.http.NvApp; import com.limelight.nvstream.input.KeyboardPacket; import com.limelight.nvstream.input.MouseButtonPacket; import com.limelight.nvstream.jni.MoonBridge; import com.limelight.preferences.GlPreferences; import com.limelight.preferences.PreferenceConfiguration; import com.limelight.ui.GameGestures; import com.limelight.ui.StreamView; import com.limelight.utils.Dialog; import com.limelight.utils.ShortcutHelper; import com.limelight.utils.SpinnerDialog; import com.limelight.utils.UiHelper; import android.annotation.SuppressLint; import android.annotation.TargetApi; import android.app.Activity; import android.app.PictureInPictureParams; import android.app.Service; import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.ServiceConnection; import android.content.SharedPreferences; import android.content.pm.ActivityInfo; import android.content.res.Configuration; import android.graphics.Point; import android.graphics.Rect; import android.hardware.input.InputManager; import android.media.AudioManager; import android.net.ConnectivityManager; import android.net.wifi.WifiManager; import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.os.IBinder; import android.os.SystemClock; import android.util.Rational; import android.view.Display; import android.view.InputDevice; import android.view.KeyEvent; import android.view.MotionEvent; import android.view.SurfaceHolder; import android.view.View; import android.view.View.OnGenericMotionListener; import android.view.View.OnSystemUiVisibilityChangeListener; import android.view.View.OnTouchListener; import android.view.Window; import android.view.WindowManager; import android.widget.FrameLayout; import android.view.inputmethod.InputMethodManager; import android.widget.Toast; import java.io.ByteArrayInputStream; import java.security.cert.CertificateException; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; import java.util.Locale; public class Game extends Activity implements SurfaceHolder.Callback, OnGenericMotionListener, OnTouchListener, NvConnectionListener, EvdevListener, OnSystemUiVisibilityChangeListener, GameGestures, StreamView.InputCallbacks { private int lastMouseX = Integer.MIN_VALUE; private int lastMouseY = Integer.MIN_VALUE; private int lastButtonState = 0; // Only 2 touches are supported private final TouchContext[] touchContextMap = new TouchContext[2]; private long threeFingerDownTime = 0; private static final int REFERENCE_HORIZ_RES = 1280; private static final int REFERENCE_VERT_RES = 720; private static final int THREE_FINGER_TAP_THRESHOLD = 300; private ControllerHandler controllerHandler; private VirtualController virtualController; private PreferenceConfiguration prefConfig; private SharedPreferences tombstonePrefs; private NvConnection conn; private SpinnerDialog spinner; private boolean displayedFailureDialog = false; private boolean connecting = false; private boolean connected = false; private boolean surfaceCreated = false; private boolean attemptedConnection = false; private InputCaptureProvider inputCaptureProvider; private int modifierFlags = 0; private boolean grabbedInput = true; private boolean grabComboDown = false; private StreamView streamView; private ShortcutHelper shortcutHelper; private MediaCodecDecoderRenderer decoderRenderer; private boolean reportedCrash; private WifiManager.WifiLock wifiLock; private boolean connectedToUsbDriverService = false; private ServiceConnection usbDriverServiceConnection = new ServiceConnection() { @Override public void onServiceConnected(ComponentName componentName, IBinder iBinder) { UsbDriverService.UsbDriverBinder binder = (UsbDriverService.UsbDriverBinder) iBinder; binder.setListener(controllerHandler); connectedToUsbDriverService = true; } @Override public void onServiceDisconnected(ComponentName componentName) { connectedToUsbDriverService = false; } }; public static final String EXTRA_HOST = "Host"; public static final String EXTRA_APP_NAME = "AppName"; public static final String EXTRA_APP_ID = "AppId"; public static final String EXTRA_UNIQUEID = "UniqueId"; public static final String EXTRA_PC_UUID = "UUID"; public static final String EXTRA_PC_NAME = "PcName"; public static final String EXTRA_APP_HDR = "HDR"; public static final String EXTRA_SERVER_CERT = "ServerCert"; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); UiHelper.setLocale(this); // We don't want a title bar requestWindowFeature(Window.FEATURE_NO_TITLE); // Full-screen getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); // If we're going to use immersive mode, we want to have // the entire screen if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.KITKAT) { getWindow().getDecorView().setSystemUiVisibility( View.SYSTEM_UI_FLAG_LAYOUT_STABLE | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN); getWindow().addFlags(WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN); } // We specified userLandscape in the manifest which isn't supported until 4.3, // so we must fall back at runtime to sensorLandscape. if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN_MR2) { setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_SENSOR_LANDSCAPE); } // Listen for UI visibility events getWindow().getDecorView().setOnSystemUiVisibilityChangeListener(this); // Change volume button behavior setVolumeControlStream(AudioManager.STREAM_MUSIC); // Inflate the content setContentView(R.layout.activity_game); // Start the spinner spinner = SpinnerDialog.displayDialog(this, getResources().getString(R.string.conn_establishing_title), getResources().getString(R.string.conn_establishing_msg), true); // Read the stream preferences prefConfig = PreferenceConfiguration.readPreferences(this); tombstonePrefs = Game.this.getSharedPreferences("DecoderTombstone", 0); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && prefConfig.stretchVideo) { // Allow the activity to layout under notches if the fill-screen option // was turned on by the user getWindow().getAttributes().layoutInDisplayCutoutMode = WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES; } // Listen for events on the game surface streamView = findViewById(R.id.surfaceView); streamView.setOnGenericMotionListener(this); streamView.setOnTouchListener(this); streamView.setInputCallbacks(this); inputCaptureProvider = InputCaptureManager.getInputCaptureProvider(this, this); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { // The view must be focusable for pointer capture to work. streamView.setFocusable(true); streamView.setDefaultFocusHighlightEnabled(false); streamView.setOnCapturedPointerListener(new View.OnCapturedPointerListener() { @Override public boolean onCapturedPointer(View view, MotionEvent motionEvent) { return handleMotionEvent(motionEvent); } }); } // Warn the user if they're on a metered connection ConnectivityManager connMgr = (ConnectivityManager) getSystemService(Context.CONNECTIVITY_SERVICE); if (connMgr.isActiveNetworkMetered()) { displayTransientMessage(getResources().getString(R.string.conn_metered)); } // Make sure Wi-Fi is fully powered up WifiManager wifiMgr = (WifiManager) getApplicationContext().getSystemService(Context.WIFI_SERVICE); wifiLock = wifiMgr.createWifiLock(WifiManager.WIFI_MODE_FULL_HIGH_PERF, "Limelight"); wifiLock.setReferenceCounted(false); wifiLock.acquire(); String host = Game.this.getIntent().getStringExtra(EXTRA_HOST); String appName = Game.this.getIntent().getStringExtra(EXTRA_APP_NAME); int appId = Game.this.getIntent().getIntExtra(EXTRA_APP_ID, StreamConfiguration.INVALID_APP_ID); String uniqueId = Game.this.getIntent().getStringExtra(EXTRA_UNIQUEID); String uuid = Game.this.getIntent().getStringExtra(EXTRA_PC_UUID); String pcName = Game.this.getIntent().getStringExtra(EXTRA_PC_NAME); boolean willStreamHdr = Game.this.getIntent().getBooleanExtra(EXTRA_APP_HDR, false); byte[] derCertData = Game.this.getIntent().getByteArrayExtra(EXTRA_SERVER_CERT); X509Certificate serverCert = null; try { if (derCertData != null) { serverCert = (X509Certificate) CertificateFactory.getInstance("X.509") .generateCertificate(new ByteArrayInputStream(derCertData)); } } catch (CertificateException e) { e.printStackTrace(); } if (appId == StreamConfiguration.INVALID_APP_ID) { finish(); return; } // Add a launcher shortcut for this PC (forced, since this is user interaction) shortcutHelper = new ShortcutHelper(this); shortcutHelper.createAppViewShortcut(uuid, pcName, uuid, true); shortcutHelper.reportShortcutUsed(uuid); // Initialize the MediaCodec helper before creating the decoder GlPreferences glPrefs = GlPreferences.readPreferences(this); MediaCodecHelper.initialize(this, glPrefs.glRenderer); // Check if the user has enabled HDR if (prefConfig.enableHdr) { // Check if the app supports it if (!willStreamHdr) { Toast.makeText(this, "This game does not support HDR10", Toast.LENGTH_SHORT).show(); } // It does, so start our HDR checklist else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { // We already know the app supports HDR if willStreamHdr is set. Display display = getWindowManager().getDefaultDisplay(); Display.HdrCapabilities hdrCaps = display.getHdrCapabilities(); // We must now ensure our display is compatible with HDR10 boolean foundHdr10 = false; for (int hdrType : hdrCaps.getSupportedHdrTypes()) { if (hdrType == Display.HdrCapabilities.HDR_TYPE_HDR10) { LimeLog.info("Display supports HDR10"); foundHdr10 = true; } } if (!foundHdr10) { // Nope, no HDR for us :( willStreamHdr = false; Toast.makeText(this, "Display does not support HDR10", Toast.LENGTH_LONG).show(); } } else { Toast.makeText(this, "HDR requires Android 7.0 or later", Toast.LENGTH_LONG).show(); willStreamHdr = false; } } else { willStreamHdr = false; } decoderRenderer = new MediaCodecDecoderRenderer(prefConfig, new CrashListener() { @Override public void notifyCrash(Exception e) { // The MediaCodec instance is going down due to a crash // let's tell the user something when they open the app again // We must use commit because the app will crash when we return from this function tombstonePrefs.edit().putInt("CrashCount", tombstonePrefs.getInt("CrashCount", 0) + 1).commit(); reportedCrash = true; } }, tombstonePrefs.getInt("CrashCount", 0), connMgr.isActiveNetworkMetered(), willStreamHdr, glPrefs.glRenderer ); // Don't stream HDR if the decoder can't support it if (willStreamHdr && !decoderRenderer.isHevcMain10Hdr10Supported()) { willStreamHdr = false; Toast.makeText(this, "Decoder does not support HEVC Main10HDR10", Toast.LENGTH_LONG).show(); } // Display a message to the user if H.265 was forced on but we still didn't find a decoder if (prefConfig.videoFormat == PreferenceConfiguration.FORCE_H265_ON && !decoderRenderer.isHevcSupported()) { Toast.makeText(this, "No H.265 decoder found.\nFalling back to H.264.", Toast.LENGTH_LONG).show(); } int gamepadMask = ControllerHandler.getAttachedControllerMask(this); if (!prefConfig.multiController) { // Always set gamepad 1 present for when multi-controller is // disabled for games that don't properly support detection // of gamepads removed and replugged at runtime. gamepadMask = 1; } if (prefConfig.onscreenController) { // If we're using OSC, always set at least gamepad 1. gamepadMask |= 1; } // Set to the optimal mode for streaming float displayRefreshRate = prepareDisplayForRendering(); LimeLog.info("Display refresh rate: "+displayRefreshRate); // HACK: Despite many efforts to ensure low latency consistent frame // delivery, the best non-lossy mechanism is to buffer 1 extra frame // in the output pipeline. Android does some buffering on its end // in SurfaceFlinger and it's difficult (impossible?) to inspect // the precise state of the buffer queue to the screen after we // release a frame for rendering. // // Since buffering a frame adds latency and we are primarily a // latency-optimized client, rather than one designed for picture-perfect // accuracy, we will synthetically induce a negative pressure on the display // output pipeline by driving the decoder input pipeline under the speed // that the display can refresh. This ensures a constant negative pressure // to keep latency down but does induce a periodic frame loss. However, this // periodic frame loss is *way* less than what we'd already get in Marshmallow's // display pipeline where frames are dropped outside of our control if they land // on the same V-sync. // // Hopefully, we can get rid of this once someone comes up with a better way // to track the state of the pipeline and time frames. int roundedRefreshRate = Math.round(displayRefreshRate); if ((!prefConfig.disableFrameDrop || prefConfig.unlockFps) && prefConfig.fps >= roundedRefreshRate) { if (prefConfig.unlockFps) { // Use frame drops when rendering above the screen frame rate decoderRenderer.enableLegacyFrameDropRendering(); LimeLog.info("Using drop mode for FPS > Hz"); } else if (roundedRefreshRate <= 49) { // Let's avoid clearly bogus refresh rates and fall back to legacy rendering decoderRenderer.enableLegacyFrameDropRendering(); LimeLog.info("Bogus refresh rate: "+roundedRefreshRate); } // HACK: Avoid crashing on some MTK devices else if (roundedRefreshRate == 50 && decoderRenderer.is49FpsBlacklisted()) { // Use the old rendering strategy on these broken devices decoderRenderer.enableLegacyFrameDropRendering(); } else { prefConfig.fps = roundedRefreshRate - 1; LimeLog.info("Adjusting FPS target for screen to "+prefConfig.fps); } } StreamConfiguration config = new StreamConfiguration.Builder() .setResolution(prefConfig.width, prefConfig.height) .setRefreshRate(prefConfig.fps) .setApp(new NvApp(appName, appId, willStreamHdr)) .setBitrate(prefConfig.bitrate) .setEnableSops(prefConfig.enableSops) .enableLocalAudioPlayback(prefConfig.playHostAudio) .setMaxPacketSize(1392) .setRemoteConfiguration(StreamConfiguration.STREAM_CFG_AUTO) .setHevcBitratePercentageMultiplier(75) .setHevcSupported(decoderRenderer.isHevcSupported()) .setEnableHdr(willStreamHdr) .setAttachedGamepadMask(gamepadMask) .setClientRefreshRateX100((int)(displayRefreshRate * 100)) .setAudioConfiguration(prefConfig.enable51Surround ? MoonBridge.AUDIO_CONFIGURATION_51_SURROUND : MoonBridge.AUDIO_CONFIGURATION_STEREO) .build(); // Initialize the connection conn = new NvConnection(host, uniqueId, config, PlatformBinding.getCryptoProvider(this), serverCert); controllerHandler = new ControllerHandler(this, conn, this, prefConfig); InputManager inputManager = (InputManager) getSystemService(Context.INPUT_SERVICE); inputManager.registerInputDeviceListener(controllerHandler, null); // Initialize touch contexts for (int i = 0; i < touchContextMap.length; i++) { touchContextMap[i] = new TouchContext(conn, i, REFERENCE_HORIZ_RES, REFERENCE_VERT_RES, streamView); } // Use sustained performance mode on N+ to ensure consistent // CPU availability if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { getWindow().setSustainedPerformanceMode(true); } if (prefConfig.onscreenController) { // create virtual onscreen controller virtualController = new VirtualController(controllerHandler, (FrameLayout)streamView.getParent(), this); virtualController.refreshLayout(); virtualController.show(); } if (prefConfig.usbDriver) { // Start the USB driver bindService(new Intent(this, UsbDriverService.class), usbDriverServiceConnection, Service.BIND_AUTO_CREATE); } if (!decoderRenderer.isAvcSupported()) { if (spinner != null) { spinner.dismiss(); spinner = null; } // If we can't find an AVC decoder, we can't proceed Dialog.displayDialog(this, getResources().getString(R.string.conn_error_title), "This device or ROM doesn't support hardware accelerated H.264 playback.", true); return; } // The connection will be started when the surface gets created streamView.getHolder().addCallback(this); } @Override public void onConfigurationChanged(Configuration newConfig) { super.onConfigurationChanged(newConfig); if (virtualController != null) { // Refresh layout of OSC for possible new screen size virtualController.refreshLayout(); // Hide OSC in PiP if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { if (isInPictureInPictureMode()) { virtualController.hide(); } else { virtualController.show(); } } } } @Override public void onUserLeaveHint() { super.onUserLeaveHint(); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { if (prefConfig.enablePip && connected) { try { // This has thrown all sorts of weird exceptions on Samsung devices // running Oreo. Just eat them and close gracefully on leave, rather // than crashing. enterPictureInPictureMode( new PictureInPictureParams.Builder() .setAspectRatio(new Rational(prefConfig.width, prefConfig.height)) .setSourceRectHint(new Rect( streamView.getLeft(), streamView.getTop(), streamView.getRight(), streamView.getBottom())) .build()); } catch (Exception e) { e.printStackTrace(); } } } } @Override public void onWindowFocusChanged(boolean hasFocus) { super.onWindowFocusChanged(hasFocus); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { // Capture is lost when focus is lost, so it must be requested again // when focus is regained. if (inputCaptureProvider.isCapturingEnabled() && hasFocus) { // Recapture the pointer if focus was regained streamView.requestPointerCapture(); } } } private float prepareDisplayForRendering() { Display display = getWindowManager().getDefaultDisplay(); WindowManager.LayoutParams windowLayoutParams = getWindow().getAttributes(); float displayRefreshRate; // On M, we can explicitly set the optimal display mode if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { Display.Mode bestMode = display.getMode(); for (Display.Mode candidate : display.getSupportedModes()) { boolean refreshRateOk = candidate.getRefreshRate() >= bestMode.getRefreshRate(); boolean resolutionOk = candidate.getPhysicalWidth() >= bestMode.getPhysicalWidth() && candidate.getPhysicalHeight() >= bestMode.getPhysicalHeight() && candidate.getPhysicalWidth() <= 4096; LimeLog.info("Examining display mode: "+candidate.getPhysicalWidth()+"x"+ candidate.getPhysicalHeight()+"x"+candidate.getRefreshRate()); // On non-4K streams, we force the resolution to never change if (prefConfig.width < 3840) { if (display.getMode().getPhysicalWidth() != candidate.getPhysicalWidth() || display.getMode().getPhysicalHeight() != candidate.getPhysicalHeight()) { continue; } } // Ensure the frame rate stays around 60 Hz for <= 60 FPS streams if (prefConfig.fps <= 60) { if (candidate.getRefreshRate() >= 63) { continue; } } // Make sure the refresh rate doesn't regress if (!refreshRateOk) { continue; } // Make sure the resolution doesn't regress if (!resolutionOk) { continue; } bestMode = candidate; } LimeLog.info("Selected display mode: "+bestMode.getPhysicalWidth()+"x"+ bestMode.getPhysicalHeight()+"x"+bestMode.getRefreshRate()); windowLayoutParams.preferredDisplayModeId = bestMode.getModeId(); displayRefreshRate = bestMode.getRefreshRate(); } // On L, we can at least tell the OS that we want a refresh rate else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { float bestRefreshRate = display.getRefreshRate(); for (float candidate : display.getSupportedRefreshRates()) { if (candidate > bestRefreshRate) { LimeLog.info("Examining refresh rate: "+candidate); // Ensure the frame rate stays around 60 Hz for <= 60 FPS streams if (prefConfig.fps <= 60) { if (candidate >= 63) { continue; } } bestRefreshRate = candidate; } } LimeLog.info("Selected refresh rate: "+bestRefreshRate); windowLayoutParams.preferredRefreshRate = bestRefreshRate; displayRefreshRate = bestRefreshRate; } else { // Otherwise, the active display refresh rate is just // whatever is currently in use. displayRefreshRate = display.getRefreshRate(); } // Apply the display mode change getWindow().setAttributes(windowLayoutParams); // From 4.4 to 5.1 we can't ask for a 4K display mode, so we'll // need to hint the OS to provide one. boolean aspectRatioMatch = false; if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT && Build.VERSION.SDK_INT <= Build.VERSION_CODES.LOLLIPOP_MR1) { // On KitKat and later (where we can use the whole screen via immersive mode), we'll // calculate whether we need to scale by aspect ratio or not. If not, we'll use // setFixedSize so we can handle 4K properly. The only known devices that have // >= 4K screens have exactly 4K screens, so we'll be able to hit this good path // on these devices. On Marshmallow, we can start changing to 4K manually but no // 4K devices run 6.0 at the moment. Point screenSize = new Point(0, 0); display.getSize(screenSize); double screenAspectRatio = ((double)screenSize.y) / screenSize.x; double streamAspectRatio = ((double)prefConfig.height) / prefConfig.width; if (Math.abs(screenAspectRatio - streamAspectRatio) < 0.001) { LimeLog.info("Stream has compatible aspect ratio with output display"); aspectRatioMatch = true; } } if (prefConfig.stretchVideo || aspectRatioMatch) { // Set the surface to the size of the video streamView.getHolder().setFixedSize(prefConfig.width, prefConfig.height); } else { // Set the surface to scale based on the aspect ratio of the stream streamView.setDesiredAspectRatio((double)prefConfig.width / (double)prefConfig.height); } return displayRefreshRate; } @SuppressLint("InlinedApi") private final Runnable hideSystemUi = new Runnable() { @Override public void run() { // In multi-window mode on N+, we need to drop our layout flags or we'll // be drawing underneath the system UI. if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && isInMultiWindowMode()) { Game.this.getWindow().getDecorView().setSystemUiVisibility( View.SYSTEM_UI_FLAG_LAYOUT_STABLE); } // Use immersive mode on 4.4+ or standard low profile on previous builds else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { Game.this.getWindow().getDecorView().setSystemUiVisibility( View.SYSTEM_UI_FLAG_LAYOUT_STABLE | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_FULLSCREEN | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY); } else { Game.this.getWindow().getDecorView().setSystemUiVisibility( View.SYSTEM_UI_FLAG_FULLSCREEN | View.SYSTEM_UI_FLAG_LOW_PROFILE); } } }; private void hideSystemUi(int delay) { Handler h = getWindow().getDecorView().getHandler(); if (h != null) { h.removeCallbacks(hideSystemUi); h.postDelayed(hideSystemUi, delay); } } @Override @TargetApi(Build.VERSION_CODES.N) public void onMultiWindowModeChanged(boolean isInMultiWindowMode) { super.onMultiWindowModeChanged(isInMultiWindowMode); // In multi-window, we don't want to use the full-screen layout // flag. It will cause us to collide with the system UI. // This function will also be called for PiP so we can cover // that case here too. if (isInMultiWindowMode) { getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); // Disable performance optimizations for foreground getWindow().setSustainedPerformanceMode(false); decoderRenderer.notifyVideoBackground(); } else { getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); // Enable performance optimizations for foreground getWindow().setSustainedPerformanceMode(true); decoderRenderer.notifyVideoForeground(); } // Correct the system UI visibility flags hideSystemUi(50); } @Override protected void onDestroy() { super.onDestroy(); if (controllerHandler != null) { InputManager inputManager = (InputManager) getSystemService(Context.INPUT_SERVICE); inputManager.unregisterInputDeviceListener(controllerHandler); } wifiLock.release(); if (connectedToUsbDriverService) { // Unbind from the discovery service unbindService(usbDriverServiceConnection); } // Destroy the capture provider inputCaptureProvider.destroy(); } @Override protected void onStop() { super.onStop(); SpinnerDialog.closeDialogs(this); Dialog.closeDialogs(); if (virtualController != null) { virtualController.hide(); } if (conn != null) { int videoFormat = decoderRenderer.getActiveVideoFormat(); displayedFailureDialog = true; stopConnection(); int averageEndToEndLat = decoderRenderer.getAverageEndToEndLatency(); int averageDecoderLat = decoderRenderer.getAverageDecoderLatency(); String message = null; if (averageEndToEndLat > 0) { message = getResources().getString(R.string.conn_client_latency)+" "+averageEndToEndLat+" ms"; if (averageDecoderLat > 0) { message += " ("+getResources().getString(R.string.conn_client_latency_hw)+" "+averageDecoderLat+" ms)"; } } else if (averageDecoderLat > 0) { message = getResources().getString(R.string.conn_hardware_latency)+" "+averageDecoderLat+" ms"; } // Add the video codec to the post-stream toast if (message != null) { if (videoFormat == MoonBridge.VIDEO_FORMAT_H265_MAIN10) { message += " [H.265 HDR]"; } else if (videoFormat == MoonBridge.VIDEO_FORMAT_H265) { message += " [H.265]"; } else if (videoFormat == MoonBridge.VIDEO_FORMAT_H264) { message += " [H.264]"; } } if (message != null) { Toast.makeText(this, message, Toast.LENGTH_LONG).show(); } // Clear the tombstone count if we terminated normally if (!reportedCrash && tombstonePrefs.getInt("CrashCount", 0) != 0) { tombstonePrefs.edit() .putInt("CrashCount", 0) .putInt("LastNotifiedCrashCount", 0) .apply(); } } finish(); } private final Runnable toggleGrab = new Runnable() { @Override public void run() { if (grabbedInput) { inputCaptureProvider.disableCapture(); } else { inputCaptureProvider.enableCapture(); } grabbedInput = !grabbedInput; } }; // Returns true if the key stroke was consumed private boolean handleSpecialKeys(int androidKeyCode, boolean down) { int modifierMask = 0; if (androidKeyCode == KeyEvent.KEYCODE_CTRL_LEFT || androidKeyCode == KeyEvent.KEYCODE_CTRL_RIGHT) { modifierMask = KeyboardPacket.MODIFIER_CTRL; } else if (androidKeyCode == KeyEvent.KEYCODE_SHIFT_LEFT || androidKeyCode == KeyEvent.KEYCODE_SHIFT_RIGHT) { modifierMask = KeyboardPacket.MODIFIER_SHIFT; } else if (androidKeyCode == KeyEvent.KEYCODE_ALT_LEFT || androidKeyCode == KeyEvent.KEYCODE_ALT_RIGHT) { modifierMask = KeyboardPacket.MODIFIER_ALT; } if (down) { this.modifierFlags |= modifierMask; } else { this.modifierFlags &= ~modifierMask; } // Check if Ctrl+Shift+Z is pressed if (androidKeyCode == KeyEvent.KEYCODE_Z && (modifierFlags & (KeyboardPacket.MODIFIER_CTRL | KeyboardPacket.MODIFIER_SHIFT)) == (KeyboardPacket.MODIFIER_CTRL | KeyboardPacket.MODIFIER_SHIFT)) { if (down) { // Now that we've pressed the magic combo // we'll wait for one of the keys to come up grabComboDown = true; } else { // Toggle the grab if Z comes up Handler h = getWindow().getDecorView().getHandler(); if (h != null) { h.postDelayed(toggleGrab, 250); } grabComboDown = false; } return true; } // Toggle the grab if control or shift comes up else if (grabComboDown) { Handler h = getWindow().getDecorView().getHandler(); if (h != null) { h.postDelayed(toggleGrab, 250); } grabComboDown = false; return true; } // Not a special combo return false; } private static byte getModifierState(KeyEvent event) { byte modifier = 0; if (event.isShiftPressed()) { modifier |= KeyboardPacket.MODIFIER_SHIFT; } if (event.isCtrlPressed()) { modifier |= KeyboardPacket.MODIFIER_CTRL; } if (event.isAltPressed()) { modifier |= KeyboardPacket.MODIFIER_ALT; } return modifier; } private byte getModifierState() { return (byte) modifierFlags; } @Override public boolean onKeyDown(int keyCode, KeyEvent event) { return handleKeyDown(event) || super.onKeyDown(keyCode, event); } @Override public boolean handleKeyDown(KeyEvent event) { // Pass-through virtual navigation keys if ((event.getFlags() & KeyEvent.FLAG_VIRTUAL_HARD_KEY) != 0) { return false; } // Handle a synthetic back button event that some Android OS versions // create as a result of a right-click. This event WILL repeat if // the right mouse button is held down, so we ignore those. if (!prefConfig.mouseNavButtons && (event.getSource() == InputDevice.SOURCE_MOUSE || event.getSource() == InputDevice.SOURCE_MOUSE_RELATIVE) && event.getKeyCode() == KeyEvent.KEYCODE_BACK) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_RIGHT); return true; } boolean handled = false; if (ControllerHandler.isGameControllerDevice(event.getDevice())) { // Always try the controller handler first, unless it's an alphanumeric keyboard device. // Otherwise, controller handler will eat keyboard d-pad events. handled = controllerHandler.handleButtonDown(event); } if (!handled) { // Try the keyboard handler short translated = KeyboardTranslator.translate(event.getKeyCode()); if (translated == 0) { return false; } // Let this method take duplicate key down events if (handleSpecialKeys(event.getKeyCode(), true)) { return true; } // Pass through keyboard input if we're not grabbing if (!grabbedInput) { return false; } byte modifiers = getModifierState(event); if (KeyboardTranslator.needsShift(event.getKeyCode())) { modifiers |= KeyboardPacket.MODIFIER_SHIFT; conn.sendKeyboardInput((short) 0x8010, KeyboardPacket.KEY_DOWN, modifiers); } conn.sendKeyboardInput(translated, KeyboardPacket.KEY_DOWN, modifiers); } return true; } @Override public boolean onKeyUp(int keyCode, KeyEvent event) { return handleKeyUp(event) || super.onKeyUp(keyCode, event); } @Override public boolean handleKeyUp(KeyEvent event) { // Pass-through virtual navigation keys if ((event.getFlags() & KeyEvent.FLAG_VIRTUAL_HARD_KEY) != 0) { return false; } // Handle a synthetic back button event that some Android OS versions // create as a result of a right-click. if (!prefConfig.mouseNavButtons && (event.getSource() == InputDevice.SOURCE_MOUSE || event.getSource() == InputDevice.SOURCE_MOUSE_RELATIVE) && event.getKeyCode() == KeyEvent.KEYCODE_BACK) { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_RIGHT); return true; } boolean handled = false; if (ControllerHandler.isGameControllerDevice(event.getDevice())) { // Always try the controller handler first, unless it's an alphanumeric keyboard device. // Otherwise, controller handler will eat keyboard d-pad events. handled = controllerHandler.handleButtonUp(event); } if (!handled) { // Try the keyboard handler short translated = KeyboardTranslator.translate(event.getKeyCode()); if (translated == 0) { return false; } if (handleSpecialKeys(event.getKeyCode(), false)) { return true; } // Pass through keyboard input if we're not grabbing if (!grabbedInput) { return false; } byte modifiers = getModifierState(event); if (KeyboardTranslator.needsShift(event.getKeyCode())) { modifiers |= KeyboardPacket.MODIFIER_SHIFT; } conn.sendKeyboardInput(translated, KeyboardPacket.KEY_UP, modifiers); if (KeyboardTranslator.needsShift(event.getKeyCode())) { conn.sendKeyboardInput((short) 0x8010, KeyboardPacket.KEY_UP, getModifierState(event)); } } return true; } private TouchContext getTouchContext(int actionIndex) { if (actionIndex < touchContextMap.length) { return touchContextMap[actionIndex]; } else { return null; } } @Override public void showKeyboard() { LimeLog.info("Showing keyboard overlay"); InputMethodManager inputManager = (InputMethodManager) getSystemService(Context.INPUT_METHOD_SERVICE); inputManager.toggleSoftInput(InputMethodManager.SHOW_FORCED, InputMethodManager.HIDE_IMPLICIT_ONLY); } // Returns true if the event was consumed private boolean handleMotionEvent(MotionEvent event) { // Pass through keyboard input if we're not grabbing if (!grabbedInput) { return false; } if ((event.getSource() & InputDevice.SOURCE_CLASS_JOYSTICK) != 0) { if (controllerHandler.handleMotionEvent(event)) { return true; } } else if ((event.getSource() & InputDevice.SOURCE_CLASS_POINTER) != 0 || event.getSource() == InputDevice.SOURCE_MOUSE_RELATIVE) { // This case is for mice if (event.getSource() == InputDevice.SOURCE_MOUSE || event.getSource() == InputDevice.SOURCE_MOUSE_RELATIVE || (event.getPointerCount() >= 1 && event.getToolType(0) == MotionEvent.TOOL_TYPE_MOUSE)) { int changedButtons = event.getButtonState() ^ lastButtonState; // Ignore mouse input if we're not capturing from our input source if (!inputCaptureProvider.isCapturingActive()) { return false; } if (event.getActionMasked() == MotionEvent.ACTION_SCROLL) { // Send the vertical scroll packet byte vScrollClicks = (byte) event.getAxisValue(MotionEvent.AXIS_VSCROLL); conn.sendMouseScroll(vScrollClicks); } else if (event.getActionMasked() == MotionEvent.ACTION_HOVER_ENTER || event.getActionMasked() == MotionEvent.ACTION_HOVER_EXIT) { // On some devices (Galaxy S8 without Oreo pointer capture), we can // get spurious ACTION_HOVER_ENTER events when right clicking with // incorrect X and Y coordinates. Just eat this event without processing it. return true; } if ((changedButtons & MotionEvent.BUTTON_PRIMARY) != 0) { if ((event.getButtonState() & MotionEvent.BUTTON_PRIMARY) != 0) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_LEFT); } else { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_LEFT); } } if ((changedButtons & MotionEvent.BUTTON_SECONDARY) != 0) { if ((event.getButtonState() & MotionEvent.BUTTON_SECONDARY) != 0) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_RIGHT); } else { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_RIGHT); } } if ((changedButtons & MotionEvent.BUTTON_TERTIARY) != 0) { if ((event.getButtonState() & MotionEvent.BUTTON_TERTIARY) != 0) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_MIDDLE); } else { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_MIDDLE); } } if (prefConfig.mouseNavButtons) { if ((changedButtons & MotionEvent.BUTTON_BACK) != 0) { if ((event.getButtonState() & MotionEvent.BUTTON_BACK) != 0) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_X1); } else { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_X1); } } if ((changedButtons & MotionEvent.BUTTON_FORWARD) != 0) { if ((event.getButtonState() & MotionEvent.BUTTON_FORWARD) != 0) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_X2); } else { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_X2); } } } // Get relative axis values if we can if (inputCaptureProvider.eventHasRelativeMouseAxes(event)) { // Send the deltas straight from the motion event conn.sendMouseMove((short) inputCaptureProvider.getRelativeAxisX(event), (short) inputCaptureProvider.getRelativeAxisY(event)); // We have to also update the position Android thinks the cursor is at // in order to avoid jumping when we stop moving or click. lastMouseX = (int)event.getX(); lastMouseY = (int)event.getY(); } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { // We get a normal (non-relative) MotionEvent when starting pointer capture to synchronize the // location of the cursor with our app. We don't want this, so we must discard this event. lastMouseX = (int)event.getX(); lastMouseY = (int)event.getY(); } else { // Don't process the history. We just want the current position now. updateMousePosition((int)event.getX(), (int)event.getY()); } lastButtonState = event.getButtonState(); } // This case is for touch-based input devices else { if (virtualController != null && virtualController.getControllerMode() == VirtualController.ControllerMode.Configuration) { // Ignore presses when the virtual controller is in configuration mode return true; } int actionIndex = event.getActionIndex(); int eventX = (int)event.getX(actionIndex); int eventY = (int)event.getY(actionIndex); // Special handling for 3 finger gesture if (event.getActionMasked() == MotionEvent.ACTION_POINTER_DOWN && event.getPointerCount() == 3) { // Three fingers down threeFingerDownTime = SystemClock.uptimeMillis(); // Cancel the first and second touches to avoid // erroneous events for (TouchContext aTouchContext : touchContextMap) { aTouchContext.cancelTouch(); } return true; } TouchContext context = getTouchContext(actionIndex); if (context == null) { return false; } switch (event.getActionMasked()) { case MotionEvent.ACTION_POINTER_DOWN: case MotionEvent.ACTION_DOWN: context.touchDownEvent(eventX, eventY); break; case MotionEvent.ACTION_POINTER_UP: case MotionEvent.ACTION_UP: if (event.getPointerCount() == 1) { // All fingers up if (SystemClock.uptimeMillis() - threeFingerDownTime < THREE_FINGER_TAP_THRESHOLD) { // This is a 3 finger tap to bring up the keyboard showKeyboard(); return true; } } context.touchUpEvent(eventX, eventY); if (actionIndex == 0 && event.getPointerCount() > 1 && !context.isCancelled()) { // The original secondary touch now becomes primary context.touchDownEvent((int)event.getX(1), (int)event.getY(1)); } break; case MotionEvent.ACTION_MOVE: // ACTION_MOVE is special because it always has actionIndex == 0 // We'll call the move handlers for all indexes manually // First process the historical events for (int i = 0; i < event.getHistorySize(); i++) { for (TouchContext aTouchContextMap : touchContextMap) { if (aTouchContextMap.getActionIndex() < event.getPointerCount()) { aTouchContextMap.touchMoveEvent( (int)event.getHistoricalX(aTouchContextMap.getActionIndex(), i), (int)event.getHistoricalY(aTouchContextMap.getActionIndex(), i)); } } } // Now process the current values for (TouchContext aTouchContextMap : touchContextMap) { if (aTouchContextMap.getActionIndex() < event.getPointerCount()) { aTouchContextMap.touchMoveEvent( (int)event.getX(aTouchContextMap.getActionIndex()), (int)event.getY(aTouchContextMap.getActionIndex())); } } break; default: return false; } } // Handled a known source return true; } // Unknown class return false; } @Override public boolean onTouchEvent(MotionEvent event) { return handleMotionEvent(event) || super.onTouchEvent(event); } @Override public boolean onGenericMotionEvent(MotionEvent event) { return handleMotionEvent(event) || super.onGenericMotionEvent(event); } private void updateMousePosition(int eventX, int eventY) { // Send a mouse move if we already have a mouse location // and the mouse coordinates change if (lastMouseX != Integer.MIN_VALUE && lastMouseY != Integer.MIN_VALUE && !(lastMouseX == eventX && lastMouseY == eventY)) { int deltaX = eventX - lastMouseX; int deltaY = eventY - lastMouseY; // Scale the deltas if the device resolution is different // than the stream resolution deltaX = (int)Math.round((double)deltaX * (REFERENCE_HORIZ_RES / (double)streamView.getWidth())); deltaY = (int)Math.round((double)deltaY * (REFERENCE_VERT_RES / (double)streamView.getHeight())); conn.sendMouseMove((short)deltaX, (short)deltaY); } // Update pointer location for delta calculation next time lastMouseX = eventX; lastMouseY = eventY; } @Override public boolean onGenericMotion(View v, MotionEvent event) { return handleMotionEvent(event); } @SuppressLint("ClickableViewAccessibility") @Override public boolean onTouch(View v, MotionEvent event) { return handleMotionEvent(event); } @Override public void stageStarting(final String stage) { runOnUiThread(new Runnable() { @Override public void run() { if (spinner != null) { spinner.setMessage(getResources().getString(R.string.conn_starting) + " " + stage); } } }); } @Override public void stageComplete(String stage) { } private void stopConnection() { if (connecting || connected) { connecting = connected = false; // Stop may take a few hundred ms to do some network I/O to tell // the server we're going away and clean up. Let it run in a separate // thread to keep things smooth for the UI. Inside moonlight-common, // we prevent another thread from starting a connection before and // during the process of stopping this one. new Thread() { public void run() { conn.stop(); } }.start(); } } @Override public void stageFailed(final String stage, final long errorCode) { runOnUiThread(new Runnable() { @Override public void run() { if (spinner != null) { spinner.dismiss(); spinner = null; } if (!displayedFailureDialog) { displayedFailureDialog = true; LimeLog.severe(stage + " failed: " + errorCode); // If video initialization failed and the surface is still valid, display extra information for the user if (stage.contains("video") && streamView.getHolder().getSurface().isValid()) { Toast.makeText(Game.this, "Video decoder failed to initialize. Your device may not support the selected resolution.", Toast.LENGTH_LONG).show(); } Dialog.displayDialog(Game.this, getResources().getString(R.string.conn_error_title), getResources().getString(R.string.conn_error_msg) + " " + stage, true); } } }); } @Override public void connectionTerminated(final long errorCode) { runOnUiThread(new Runnable() { @Override public void run() { // Let the display go to sleep now getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); // Enable cursor visibility again inputCaptureProvider.disableCapture(); if (!displayedFailureDialog) { displayedFailureDialog = true; LimeLog.severe("Connection terminated: " + errorCode); stopConnection(); // Display the error dialog if it was an unexpected termination. // Otherwise, just finish the activity immediately. if (errorCode != 0) { Dialog.displayDialog(Game.this, getResources().getString(R.string.conn_terminated_title), getResources().getString(R.string.conn_terminated_msg), true); } else { finish(); } } } }); } @Override public void connectionStarted() { runOnUiThread(new Runnable() { @Override public void run() { if (spinner != null) { spinner.dismiss(); spinner = null; } connected = true; connecting = false; // Hide the mouse cursor now. Doing it before // dismissing the spinner seems to be undone // when the spinner gets displayed. inputCaptureProvider.enableCapture(); // Keep the display on getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); hideSystemUi(1000); } }); } @Override public void displayMessage(final String message) { runOnUiThread(new Runnable() { @Override public void run() { Toast.makeText(Game.this, message, Toast.LENGTH_LONG).show(); } }); } @Override public void displayTransientMessage(final String message) { if (!prefConfig.disableWarnings) { runOnUiThread(new Runnable() { @Override public void run() { Toast.makeText(Game.this, message, Toast.LENGTH_LONG).show(); } }); } } @Override public void rumble(short controllerNumber, short lowFreqMotor, short highFreqMotor) { LimeLog.info(String.format((Locale)null, "Rumble on gamepad %d: %04x %04x", controllerNumber, lowFreqMotor, highFreqMotor)); controllerHandler.handleRumble(controllerNumber, lowFreqMotor, highFreqMotor); } @Override public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { if (!surfaceCreated) { throw new IllegalStateException("Surface changed before creation!"); } if (!attemptedConnection) { attemptedConnection = true; decoderRenderer.setRenderTarget(holder); conn.start(PlatformBinding.getAudioRenderer(), decoderRenderer, Game.this); } } @Override public void surfaceCreated(SurfaceHolder holder) { surfaceCreated = true; } @Override public void surfaceDestroyed(SurfaceHolder holder) { if (!surfaceCreated) { throw new IllegalStateException("Surface destroyed before creation!"); } if (attemptedConnection) { // Let the decoder know immediately that the surface is gone decoderRenderer.prepareForStop(); if (connected) { stopConnection(); } } } @Override public void mouseMove(int deltaX, int deltaY) { conn.sendMouseMove((short) deltaX, (short) deltaY); } @Override public void mouseButtonEvent(int buttonId, boolean down) { byte buttonIndex; switch (buttonId) { case EvdevListener.BUTTON_LEFT: buttonIndex = MouseButtonPacket.BUTTON_LEFT; break; case EvdevListener.BUTTON_MIDDLE: buttonIndex = MouseButtonPacket.BUTTON_MIDDLE; break; case EvdevListener.BUTTON_RIGHT: buttonIndex = MouseButtonPacket.BUTTON_RIGHT; break; case EvdevListener.BUTTON_X1: buttonIndex = MouseButtonPacket.BUTTON_X1; break; case EvdevListener.BUTTON_X2: buttonIndex = MouseButtonPacket.BUTTON_X2; break; default: LimeLog.warning("Unhandled button: "+buttonId); return; } if (down) { conn.sendMouseButtonDown(buttonIndex); } else { conn.sendMouseButtonUp(buttonIndex); } } @Override public void mouseScroll(byte amount) { conn.sendMouseScroll(amount); } @Override public void keyboardEvent(boolean buttonDown, short keyCode) { short keyMap = KeyboardTranslator.translate(keyCode); if (keyMap != 0) { // handleSpecialKeys() takes the Android keycode if (handleSpecialKeys(keyCode, buttonDown)) { return; } if (buttonDown) { conn.sendKeyboardInput(keyMap, KeyboardPacket.KEY_DOWN, getModifierState()); } else { conn.sendKeyboardInput(keyMap, KeyboardPacket.KEY_UP, getModifierState()); } } } @Override public void onSystemUiVisibilityChange(int visibility) { // Don't do anything if we're not connected if (!connected) { return; } // This flag is set for all devices if ((visibility & View.SYSTEM_UI_FLAG_FULLSCREEN) == 0) { hideSystemUi(2000); } // This flag is only set on 4.4+ else if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.KITKAT && (visibility & View.SYSTEM_UI_FLAG_HIDE_NAVIGATION) == 0) { hideSystemUi(2000); } // This flag is only set before 4.4+ else if (android.os.Build.VERSION.SDK_INT < android.os.Build.VERSION_CODES.KITKAT && (visibility & View.SYSTEM_UI_FLAG_LOW_PROFILE) == 0) { hideSystemUi(2000); } } }