2020-11-01 20:00:39 -06:00

631 lines
24 KiB
Objective-C

//
// StreamView.m
// Moonlight
//
// Created by Cameron Gutman on 10/19/14.
// Copyright (c) 2014 Moonlight Stream. All rights reserved.
//
#import "StreamView.h"
#include <Limelight.h>
#import "DataManager.h"
#import "ControllerSupport.h"
#import "KeyboardSupport.h"
#import "RelativeTouchHandler.h"
#import "AbsoluteTouchHandler.h"
static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
@implementation StreamView {
OnScreenControls* onScreenControls;
UITextField* keyInputField;
BOOL isInputingText;
float streamAspectRatio;
// iOS 13.4 mouse support
NSInteger lastMouseButtonMask;
float lastMouseX;
float lastMouseY;
// Citrix X1 mouse support
X1Mouse* x1mouse;
double accumulatedMouseDeltaX;
double accumulatedMouseDeltaY;
UIResponder* touchHandler;
id<UserInteractionDelegate> interactionDelegate;
NSTimer* interactionTimer;
BOOL hasUserInteracted;
NSDictionary<NSString *, NSNumber *> *dictCodes;
}
- (void) setupStreamView:(ControllerSupport*)controllerSupport
swipeDelegate:(id<EdgeDetectionDelegate>)swipeDelegate
interactionDelegate:(id<UserInteractionDelegate>)interactionDelegate
config:(StreamConfiguration*)streamConfig {
self->interactionDelegate = interactionDelegate;
self->streamAspectRatio = (float)streamConfig.width / (float)streamConfig.height;
TemporarySettings* settings = [[[DataManager alloc] init] getSettings];
keyInputField = [[UITextField alloc] initWithFrame:CGRectZero];
[self addSubview:keyInputField];
#if TARGET_OS_TV
// tvOS requires RelativeTouchHandler to manage Apple Remote input
self->touchHandler = [[RelativeTouchHandler alloc] initWithView:self];
#else
// iOS uses RelativeTouchHandler or AbsoluteTouchHandler depending on user preference
if (settings.absoluteTouchMode) {
self->touchHandler = [[AbsoluteTouchHandler alloc] initWithView:self];
}
else {
self->touchHandler = [[RelativeTouchHandler alloc] initWithView:self];
}
onScreenControls = [[OnScreenControls alloc] initWithView:self controllerSup:controllerSupport swipeDelegate:swipeDelegate];
OnScreenControlsLevel level = (OnScreenControlsLevel)[settings.onscreenControls integerValue];
if (settings.absoluteTouchMode) {
Log(LOG_I, @"On-screen controls disabled in absolute touch mode");
[onScreenControls setLevel:OnScreenControlsLevelOff];
}
else if (level == OnScreenControlsLevelAuto) {
[controllerSupport initAutoOnScreenControlMode:onScreenControls];
}
else {
Log(LOG_I, @"Setting manual on-screen controls level: %d", (int)level);
[onScreenControls setLevel:level];
}
// It would be nice to just use GCMouse on iOS 14+ and the older API on iOS 13
// but unfortunately that isn't possible today. GCMouse doesn't recognize many
// mice correctly, but UIKit does. We will register for both and ignore UIKit
// events if a GCMouse is connected.
if (@available(iOS 13.4, *)) {
[self addInteraction:[[UIPointerInteraction alloc] initWithDelegate:self]];
UIPanGestureRecognizer *mouseWheelRecognizer = [[UIPanGestureRecognizer alloc] initWithTarget:self action:@selector(mouseWheelMoved:)];
mouseWheelRecognizer.allowedScrollTypesMask = UIScrollTypeMaskAll;
mouseWheelRecognizer.allowedTouchTypes = @[@(UITouchTypeIndirectPointer)];
[self addGestureRecognizer:mouseWheelRecognizer];
}
#endif
x1mouse = [[X1Mouse alloc] init];
x1mouse.delegate = self;
if (settings.btMouseSupport) {
[x1mouse start];
}
}
- (void)startInteractionTimer {
// Restart user interaction tracking
hasUserInteracted = NO;
BOOL timerAlreadyRunning = interactionTimer != nil;
// Start/restart the timer
[interactionTimer invalidate];
interactionTimer = [NSTimer scheduledTimerWithTimeInterval:2.0
target:self
selector:@selector(interactionTimerExpired:)
userInfo:nil
repeats:NO];
// Notify the delegate if this was a new user interaction
if (!timerAlreadyRunning) {
[interactionDelegate userInteractionBegan];
}
}
- (void)interactionTimerExpired:(NSTimer *)timer {
if (!hasUserInteracted) {
// User has finished touching the screen
interactionTimer = nil;
[interactionDelegate userInteractionEnded];
}
else {
// User is still touching the screen. Restart the timer.
[self startInteractionTimer];
}
}
- (void) showOnScreenControls {
#if !TARGET_OS_TV
[onScreenControls show];
[self becomeFirstResponder];
#endif
}
- (OnScreenControlsLevel) getCurrentOscState {
if (onScreenControls == nil) {
return OnScreenControlsLevelOff;
}
else {
return [onScreenControls getLevel];
}
}
- (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
if ([self handleMouseButtonEvent:BUTTON_ACTION_PRESS
forTouches:touches
withEvent:event]) {
// If it's a mouse event, we're done
return;
}
Log(LOG_D, @"Touch down");
// Notify of user interaction and start expiration timer
[self startInteractionTimer];
if (![onScreenControls handleTouchDownEvent:touches]) {
if ([[event allTouches] count] == 3) {
if (isInputingText) {
Log(LOG_D, @"Closing the keyboard");
[keyInputField resignFirstResponder];
isInputingText = false;
} else {
Log(LOG_D, @"Opening the keyboard");
// Prepare the textbox used to capture keyboard events.
keyInputField.delegate = self;
keyInputField.text = @"0";
[keyInputField becomeFirstResponder];
[keyInputField addTarget:self action:@selector(onKeyboardPressed:) forControlEvents:UIControlEventEditingChanged];
// Undo causes issues for our state management, so turn it off
[keyInputField.undoManager disableUndoRegistration];
isInputingText = true;
}
}
else {
[touchHandler touchesBegan:touches withEvent:event];
}
}
}
- (BOOL)handleMouseButtonEvent:(int)buttonAction forTouches:(NSSet *)touches withEvent:(UIEvent *)event {
#if !TARGET_OS_TV
if (@available(iOS 13.4, *)) {
UITouch* touch = [touches anyObject];
if (touch.type == UITouchTypeIndirectPointer) {
if (@available(iOS 14.0, *)) {
if ([GCMouse current] != nil) {
// We'll handle this with GCMouse. Do nothing here.
return YES;
}
}
UIEventButtonMask normalizedButtonMask;
// iOS 14 includes the released button in the buttonMask for the release
// event, while iOS 13 does not. Normalize that behavior here.
if (@available(iOS 14.0, *)) {
if (buttonAction == BUTTON_ACTION_RELEASE) {
normalizedButtonMask = lastMouseButtonMask & ~event.buttonMask;
}
else {
normalizedButtonMask = event.buttonMask;
}
}
else {
normalizedButtonMask = event.buttonMask;
}
UIEventButtonMask changedButtons = lastMouseButtonMask ^ normalizedButtonMask;
for (int i = BUTTON_LEFT; i <= BUTTON_X2; i++) {
UIEventButtonMask buttonFlag;
switch (i) {
// Right and Middle are reversed from what iOS uses
case BUTTON_RIGHT:
buttonFlag = UIEventButtonMaskForButtonNumber(2);
break;
case BUTTON_MIDDLE:
buttonFlag = UIEventButtonMaskForButtonNumber(3);
break;
default:
buttonFlag = UIEventButtonMaskForButtonNumber(i);
break;
}
if (changedButtons & buttonFlag) {
LiSendMouseButtonEvent(buttonAction, i);
}
}
lastMouseButtonMask = normalizedButtonMask;
return YES;
}
}
#endif
return NO;
}
- (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event {
#if !TARGET_OS_TV
if (@available(iOS 13.4, *)) {
UITouch *touch = [touches anyObject];
if (touch.type == UITouchTypeIndirectPointer) {
if (@available(iOS 14.0, *)) {
if ([GCMouse current] != nil) {
// We'll handle this with GCMouse. Do nothing here.
return;
}
}
// We must handle this event to properly support
// drags while the middle, X1, or X2 mouse buttons are
// held down. For some reason, left and right buttons
// don't require this, but we do it anyway for them too.
// Cursor movement without a button held down is handled
// in pointerInteraction:regionForRequest:defaultRegion.
[self updateCursorLocation:[touch locationInView:self] isMouse:YES];
return;
}
}
#endif
hasUserInteracted = YES;
if (![onScreenControls handleTouchMovedEvent:touches]) {
[touchHandler touchesMoved:touches withEvent:event];
}
}
- (void)pressesBegan:(NSSet<UIPress *> *)presses withEvent:(UIPressesEvent *)event {
BOOL handled = NO;
if (@available(iOS 13.4, tvOS 13.4, *)) {
for (UIPress* press in presses) {
// For now, we'll treated it as handled if we handle at least one of the
// UIPress events inside the set.
if (press.key != nil && [KeyboardSupport sendKeyEvent:press.key down:YES]) {
// This will prevent the legacy UITextField from receiving the event
handled = YES;
}
}
}
if (!handled) {
[super pressesBegan:presses withEvent:event];
}
}
- (void)pressesEnded:(NSSet<UIPress *> *)presses withEvent:(UIPressesEvent *)event {
BOOL handled = NO;
if (@available(iOS 13.4, tvOS 13.4, *)) {
for (UIPress* press in presses) {
// For now, we'll treated it as handled if we handle at least one of the
// UIPress events inside the set.
if (press.key != nil && [KeyboardSupport sendKeyEvent:press.key down:NO]) {
// This will prevent the legacy UITextField from receiving the event
handled = YES;
}
}
}
if (!handled) {
[super pressesEnded:presses withEvent:event];
}
}
- (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
if ([self handleMouseButtonEvent:BUTTON_ACTION_RELEASE
forTouches:touches
withEvent:event]) {
// If it's a mouse event, we're done
return;
}
Log(LOG_D, @"Touch up");
hasUserInteracted = YES;
if (![onScreenControls handleTouchUpEvent:touches]) {
[touchHandler touchesEnded:touches withEvent:event];
}
}
- (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event {
[touchHandler touchesCancelled:touches withEvent:event];
[self handleMouseButtonEvent:BUTTON_ACTION_RELEASE
forTouches:touches
withEvent:event];
}
#if !TARGET_OS_TV
- (void) updateCursorLocation:(CGPoint)location isMouse:(BOOL)isMouse {
// These are now relative to the StreamView, however we need to scale them
// further to make them relative to the actual video portion.
float x = location.x - self.bounds.origin.x;
float y = location.y - self.bounds.origin.y;
// For some reason, we don't seem to always get to the bounds of the window
// so we'll subtract 1 pixel if we're to the left/below of the origin and
// and add 1 pixel if we're to the right/above. It should be imperceptible
// to the user but it will allow activation of gestures that require contact
// with the edge of the screen (like Aero Snap).
if (x < self.bounds.size.width / 2) {
x--;
}
else {
x++;
}
if (y < self.bounds.size.height / 2) {
y--;
}
else {
y++;
}
// This logic mimics what iOS does with AVLayerVideoGravityResizeAspect
CGSize videoSize;
CGPoint videoOrigin;
if (self.bounds.size.width > self.bounds.size.height * streamAspectRatio) {
videoSize = CGSizeMake(self.bounds.size.height * streamAspectRatio, self.bounds.size.height);
} else {
videoSize = CGSizeMake(self.bounds.size.width, self.bounds.size.width / streamAspectRatio);
}
videoOrigin = CGPointMake(self.bounds.size.width / 2 - videoSize.width / 2,
self.bounds.size.height / 2 - videoSize.height / 2);
// Confine the cursor to the video region. We don't just discard events outside
// the region because we won't always get one exactly when the mouse leaves the region.
x = MIN(MAX(x, videoOrigin.x), videoOrigin.x + videoSize.width);
y = MIN(MAX(y, videoOrigin.y), videoOrigin.y + videoSize.height);
// Send the mouse position relative to the video region if it has changed
// if we're receiving coordinates from a real mouse.
//
// NB: It is important for functionality (not just optimization) to only
// send it if the value has changed. We will receive one of these events
// any time the user presses a modifier key, which can result in errant
// mouse motion when using a Citrix X1 mouse.
if (x != lastMouseX || y != lastMouseY || !isMouse) {
if (lastMouseX != 0 || lastMouseY != 0 || !isMouse) {
LiSendMousePositionEvent(x - videoOrigin.x, y - videoOrigin.y,
videoSize.width, videoSize.height);
}
if (isMouse) {
lastMouseX = x;
lastMouseY = y;
}
}
}
- (UIPointerRegion *)pointerInteraction:(UIPointerInteraction *)interaction
regionForRequest:(UIPointerRegionRequest *)request
defaultRegion:(UIPointerRegion *)defaultRegion API_AVAILABLE(ios(13.4)) {
if (@available(iOS 14.0, *)) {
if ([GCMouse current] != nil) {
// We'll handle this with GCMouse. Do nothing here.
return nil;
}
}
// This logic mimics what iOS does with AVLayerVideoGravityResizeAspect
CGSize videoSize;
CGPoint videoOrigin;
if (self.bounds.size.width > self.bounds.size.height * streamAspectRatio) {
videoSize = CGSizeMake(self.bounds.size.height * streamAspectRatio, self.bounds.size.height);
} else {
videoSize = CGSizeMake(self.bounds.size.width, self.bounds.size.width / streamAspectRatio);
}
videoOrigin = CGPointMake(self.bounds.size.width / 2 - videoSize.width / 2,
self.bounds.size.height / 2 - videoSize.height / 2);
// Move the cursor on the host if no buttons are pressed.
// Motion with buttons pressed in handled in touchesMoved:
if (lastMouseButtonMask == 0) {
[self updateCursorLocation:request.location isMouse:YES];
}
// The pointer interaction should cover the video region only
return [UIPointerRegion regionWithRect:CGRectMake(videoOrigin.x, videoOrigin.y, videoSize.width, videoSize.height) identifier:nil];
}
- (UIPointerStyle *)pointerInteraction:(UIPointerInteraction *)interaction styleForRegion:(UIPointerRegion *)region API_AVAILABLE(ios(13.4)) {
// Always hide the mouse cursor over our stream view
return [UIPointerStyle hiddenPointerStyle];
}
- (void)mouseWheelMoved:(UIPanGestureRecognizer *)gesture {
if (@available(iOS 14.0, *)) {
if ([GCMouse current] != nil) {
// We'll handle this with GCMouse. Do nothing here.
return;
}
}
switch (gesture.state) {
case UIGestureRecognizerStateBegan:
case UIGestureRecognizerStateChanged:
case UIGestureRecognizerStateEnded:
break;
default:
// Ignore recognition failure and other states
return;
}
CGPoint velocity = [gesture velocityInView:self];
if ((short)velocity.y != 0) {
LiSendHighResScrollEvent((short)velocity.y);
}
}
#endif
- (BOOL)textFieldShouldReturn:(UITextField *)textField {
// This method is called when the "Return" key is pressed.
LiSendKeyboardEvent(0x0d, KEY_ACTION_DOWN, 0);
usleep(50 * 1000);
LiSendKeyboardEvent(0x0d, KEY_ACTION_UP, 0);
return NO;
}
- (void)onKeyboardPressed:(UITextField *)textField {
NSString* inputText = textField.text;
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{
// If the text became empty, we know the user pressed the backspace key.
if ([inputText isEqual:@""]) {
LiSendKeyboardEvent(0x08, KEY_ACTION_DOWN, 0);
usleep(50 * 1000);
LiSendKeyboardEvent(0x08, KEY_ACTION_UP, 0);
} else {
// Character 0 will be our known sentinel value
for (int i = 1; i < [inputText length]; i++) {
struct KeyEvent event = [KeyboardSupport translateKeyEvent:[inputText characterAtIndex:i] withModifierFlags:0];
if (event.keycode == 0) {
// If we don't know the code, don't send anything.
Log(LOG_W, @"Unknown key code: [%c]", [inputText characterAtIndex:i]);
continue;
}
[self sendLowLevelEvent:event];
}
}
});
// Reset text field back to known state
textField.text = @"0";
// Move the insertion point back to the end of the text box
UITextRange *textRange = [textField textRangeFromPosition:textField.endOfDocument toPosition:textField.endOfDocument];
[textField setSelectedTextRange:textRange];
}
- (void)specialCharPressed:(UIKeyCommand *)cmd {
struct KeyEvent event = [KeyboardSupport translateKeyEvent:0x20 withModifierFlags:[cmd modifierFlags]];
event.keycode = [[dictCodes valueForKey:[cmd input]] intValue];
[self sendLowLevelEvent:event];
}
- (void)keyPressed:(UIKeyCommand *)cmd {
struct KeyEvent event = [KeyboardSupport translateKeyEvent:[[cmd input] characterAtIndex:0] withModifierFlags:[cmd modifierFlags]];
[self sendLowLevelEvent:event];
}
- (void)sendLowLevelEvent:(struct KeyEvent)event {
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{
// When we want to send a modified key (like uppercase letters) we need to send the
// modifier ("shift") seperately from the key itself.
if (event.modifier != 0) {
LiSendKeyboardEvent(event.modifierKeycode, KEY_ACTION_DOWN, event.modifier);
}
LiSendKeyboardEvent(event.keycode, KEY_ACTION_DOWN, event.modifier);
usleep(50 * 1000);
LiSendKeyboardEvent(event.keycode, KEY_ACTION_UP, event.modifier);
if (event.modifier != 0) {
LiSendKeyboardEvent(event.modifierKeycode, KEY_ACTION_UP, event.modifier);
}
});
}
- (BOOL)canBecomeFirstResponder {
return YES;
}
- (NSArray<UIKeyCommand *> *)keyCommands
{
NSString *charset = @"qwertyuiopasdfghjklzxcvbnm1234567890\t§[]\\'\"/.,`<>-´ç+`¡'º;ñ= ";
NSMutableArray<UIKeyCommand *> * commands = [NSMutableArray<UIKeyCommand *> array];
dictCodes = [[NSDictionary alloc] initWithObjectsAndKeys: [NSNumber numberWithInt: 0x0d], @"\r", [NSNumber numberWithInt: 0x08], @"\b", [NSNumber numberWithInt: 0x1b], UIKeyInputEscape, [NSNumber numberWithInt: 0x28], UIKeyInputDownArrow, [NSNumber numberWithInt: 0x26], UIKeyInputUpArrow, [NSNumber numberWithInt: 0x25], UIKeyInputLeftArrow, [NSNumber numberWithInt: 0x27], UIKeyInputRightArrow, nil];
[charset enumerateSubstringsInRange:NSMakeRange(0, charset.length)
options:NSStringEnumerationByComposedCharacterSequences
usingBlock:^(NSString *substring, NSRange substringRange, NSRange enclosingRange, BOOL *stop) {
[commands addObject:[UIKeyCommand keyCommandWithInput:substring modifierFlags:0 action:@selector(keyPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:substring modifierFlags:UIKeyModifierShift action:@selector(keyPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:substring modifierFlags:UIKeyModifierControl action:@selector(keyPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:substring modifierFlags:UIKeyModifierAlternate action:@selector(keyPressed:)]];
}];
for (NSString *c in [dictCodes keyEnumerator]) {
[commands addObject:[UIKeyCommand keyCommandWithInput:c
modifierFlags:0
action:@selector(specialCharPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:c
modifierFlags:UIKeyModifierShift
action:@selector(specialCharPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:c
modifierFlags:UIKeyModifierShift | UIKeyModifierAlternate
action:@selector(specialCharPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:c
modifierFlags:UIKeyModifierShift | UIKeyModifierControl
action:@selector(specialCharPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:c
modifierFlags:UIKeyModifierControl
action:@selector(specialCharPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:c
modifierFlags:UIKeyModifierControl | UIKeyModifierAlternate
action:@selector(specialCharPressed:)]];
[commands addObject:[UIKeyCommand keyCommandWithInput:c
modifierFlags:UIKeyModifierAlternate
action:@selector(specialCharPressed:)]];
}
return commands;
}
- (void)connectedStateDidChangeWithIdentifier:(NSUUID * _Nonnull)identifier isConnected:(BOOL)isConnected {
NSLog(@"Citrix X1 mouse state change: %@ -> %s",
identifier, isConnected ? "connected" : "disconnected");
}
- (void)mouseDidMoveWithIdentifier:(NSUUID * _Nonnull)identifier deltaX:(int16_t)deltaX deltaY:(int16_t)deltaY {
accumulatedMouseDeltaX += deltaX / X1_MOUSE_SPEED_DIVISOR;
accumulatedMouseDeltaY += deltaY / X1_MOUSE_SPEED_DIVISOR;
short shortX = (short)accumulatedMouseDeltaX;
short shortY = (short)accumulatedMouseDeltaY;
if (shortX == 0 && shortY == 0) {
return;
}
LiSendMouseMoveEvent(shortX, shortY);
accumulatedMouseDeltaX -= shortX;
accumulatedMouseDeltaY -= shortY;
}
- (int) buttonFromX1ButtonCode:(enum X1MouseButton)button {
switch (button) {
case X1MouseButtonLeft:
return BUTTON_LEFT;
case X1MouseButtonRight:
return BUTTON_RIGHT;
case X1MouseButtonMiddle:
return BUTTON_MIDDLE;
default:
return -1;
}
}
- (void)mouseDownWithIdentifier:(NSUUID * _Nonnull)identifier button:(enum X1MouseButton)button {
LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, [self buttonFromX1ButtonCode:button]);
}
- (void)mouseUpWithIdentifier:(NSUUID * _Nonnull)identifier button:(enum X1MouseButton)button {
LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, [self buttonFromX1ButtonCode:button]);
}
- (void)wheelDidScrollWithIdentifier:(NSUUID * _Nonnull)identifier deltaZ:(int8_t)deltaZ {
LiSendScrollEvent(deltaZ);
}
@end