FreeRDP
Loading...
Searching...
No Matches
RDPSessionView.m
1/*
2 RDP Session View
3
4 Copyright 2013 Thincast Technologies GmbH, Author: Martin Fleisz
5
6 This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
7 If a copy of the MPL was not distributed with this file, You can obtain one at
8 http://mozilla.org/MPL/2.0/.
9 */
10
11#import "RDPSessionView.h"
12#import "RDPCursor.h"
13#import "RDPKeyboard.h"
14
15#include <winpr/wtypes.h>
16#include <freerdp/locale/keyboard.h>
17
18#import <Metal/Metal.h>
19#import <QuartzCore/CAMetalLayer.h>
20
21@class RDPSessionView;
22
23@interface RDPSessionDisplayLinkTarget : NSObject
24{
25 RDPSessionView *_view;
26}
27- (id)initWithView:(RDPSessionView *)view;
28- (void)displayLinkDidFire:(CADisplayLink *)displayLink;
29- (void)clearView;
30@end
31
32@interface RDPSessionView ()
33{
34 id<MTLDevice> _metalDevice;
35 id<MTLCommandQueue> _commandQueue;
36 id<MTLRenderPipelineState> _pipelineState;
37 id<MTLTexture> _desktopTexture;
38 CADisplayLink *_displayLink;
39 RDPSessionDisplayLinkTarget *_displayLinkTarget;
40 CGRect _pendingDirtyRect;
41 BOOL _hasPendingDirtyRect;
42 CGSize _drawableSize;
43 UIImageView *_cursorImageView;
44 CGPoint _cursorPosition;
45 CGPoint _cursorHotspot;
46}
47
48- (void)configureMetal;
49- (void)configureDesktopTexture;
50- (void)scheduleRender;
51- (void)renderDisplayLink:(CADisplayLink *)displayLink;
52@end
53
54@implementation RDPSessionDisplayLinkTarget
55
56- (id)initWithView:(RDPSessionView *)view
57{
58 self = [super init];
59 if (self)
60 _view = view;
61 return self;
62}
63
64- (void)displayLinkDidFire:(CADisplayLink *)displayLink
65{
66 [_view renderDisplayLink:displayLink];
67}
68
69- (void)clearView
70{
71 _view = nil;
72}
73
74@end
75
76@implementation RDPSessionView
77
78+ (Class)layerClass
79{
80 return [CAMetalLayer class];
81}
82
83- (void)setSession:(RDPSession *)session
84{
85 _session = session;
86 [self configureDesktopTexture];
87 [self setNeedsDisplayInRemoteRect:[self bounds]];
88}
89
90- (void)awakeFromNib
91{
92 [super awakeFromNib];
93 // The view is kept at the remote desktop's native size. UIScrollView applies
94 // the final device-size transform without creating a Retina-sized drawable.
95 [self setContentScaleFactor:1.0f];
96 [self setOpaque:YES];
97 [self setBackgroundColor:[UIColor blackColor]];
98 [self setClipsToBounds:YES];
99 _session = nil;
100 _pendingDirtyRect = CGRectNull;
101 _hasPendingDirtyRect = NO;
102 _drawableSize = CGSizeZero;
103 _cursorPosition = CGPointZero;
104 _cursorHotspot = CGPointZero;
105 _cursorImageView = [[UIImageView alloc] initWithFrame:CGRectZero];
106 [_cursorImageView setUserInteractionEnabled:NO];
107 [_cursorImageView setContentMode:UIViewContentModeTopLeft];
108 [[_cursorImageView layer] setMagnificationFilter:kCAFilterNearest];
109 [[_cursorImageView layer] setMinificationFilter:kCAFilterNearest];
110 [_cursorImageView setHidden:YES];
111 [self addSubview:_cursorImageView];
112 [self configureMetal];
113}
114
115- (void)updateRemoteCursorFrame
116{
117 UIImage *image = [_cursorImageView image];
118 if (!image)
119 return;
120
121 [_cursorImageView setFrame:CGRectMake(_cursorPosition.x - _cursorHotspot.x,
122 _cursorPosition.y - _cursorHotspot.y, [image size].width,
123 [image size].height)];
124 [self bringSubviewToFront:_cursorImageView];
125}
126
127- (void)setRemoteCursor:(RDPCursor *)cursor
128{
129 if (!cursor || ![cursor image])
130 {
131 [self hideRemoteCursor];
132 return;
133 }
134
135 _cursorHotspot = [cursor hotspot];
136 [_cursorImageView setImage:[cursor image]];
137 [_cursorImageView setHidden:NO];
138 [self updateRemoteCursorFrame];
139}
140
141- (void)setRemoteCursorPosition:(CGPoint)position
142{
143 _cursorPosition = position;
144 [self updateRemoteCursorFrame];
145}
146
147- (void)showRemoteCursor
148{
149 if ([_cursorImageView image])
150 [_cursorImageView setHidden:NO];
151}
152
153- (void)hideRemoteCursor
154{
155 [_cursorImageView setHidden:YES];
156}
157
158- (void)setDefaultRemoteCursor
159{
160 [self setRemoteCursor:[RDPCursor defaultCursor]];
161}
162
163- (void)configureMetal
164{
165 _metalDevice = [MTLCreateSystemDefaultDevice() retain];
166 if (!_metalDevice)
167 return;
168
169 CAMetalLayer *metalLayer = (CAMetalLayer *)[self layer];
170 [metalLayer setDevice:_metalDevice];
171 [metalLayer setPixelFormat:MTLPixelFormatBGRA8Unorm];
172 [metalLayer setFramebufferOnly:YES];
173 [metalLayer setOpaque:YES];
174 [metalLayer setContentsScale:1.0f];
175
176 _commandQueue = [_metalDevice newCommandQueue];
177 NSString *shaderSource =
178 @"#include <metal_stdlib>\n"
179 "using namespace metal;\n"
180 "struct VertexOut { float4 position [[position]]; float2 texCoord; };\n"
181 "vertex VertexOut desktopVertex(uint vertexID [[vertex_id]]) {\n"
182 " const float2 positions[4] = { float2(-1.0, 1.0), float2(-1.0, -1.0), "
183 "float2(1.0, 1.0), float2(1.0, -1.0) };\n"
184 " const float2 texCoords[4] = { float2(0.0, 0.0), float2(0.0, 1.0), "
185 "float2(1.0, 0.0), float2(1.0, 1.0) };\n"
186 " VertexOut out; out.position = float4(positions[vertexID], 0.0, 1.0); "
187 "out.texCoord = texCoords[vertexID]; return out;\n"
188 "}\n"
189 "fragment float4 desktopFragment(VertexOut in [[stage_in]], "
190 "texture2d<float> desktop [[texture(0)]]) {\n"
191 " constexpr sampler textureSampler(address::clamp_to_edge, filter::linear);\n"
192 " float4 color = desktop.sample(textureSampler, in.texCoord);\n"
193 " return float4(color.rgb, 1.0);\n"
194 "}\n";
195
196 NSError *error = nil;
197 id<MTLLibrary> library = [_metalDevice newLibraryWithSource:shaderSource
198 options:nil
199 error:&error];
200 if (!library)
201 {
202 NSLog(@"Failed to create iFreeRDP Metal shader library: %@", error);
203 return;
204 }
205
206 id<MTLFunction> vertexFunction = [library newFunctionWithName:@"desktopVertex"];
207 id<MTLFunction> fragmentFunction = [library newFunctionWithName:@"desktopFragment"];
208 MTLRenderPipelineDescriptor *descriptor =
209 [[[MTLRenderPipelineDescriptor alloc] init] autorelease];
210 [descriptor setVertexFunction:vertexFunction];
211 [descriptor setFragmentFunction:fragmentFunction];
212 [[descriptor colorAttachments][0] setPixelFormat:MTLPixelFormatBGRA8Unorm];
213 _pipelineState = [_metalDevice newRenderPipelineStateWithDescriptor:descriptor error:&error];
214 [vertexFunction release];
215 [fragmentFunction release];
216 [library release];
217
218 if (!_pipelineState)
219 {
220 NSLog(@"Failed to create iFreeRDP Metal pipeline: %@", error);
221 return;
222 }
223
224 _displayLinkTarget = [[RDPSessionDisplayLinkTarget alloc] initWithView:self];
225 _displayLink = [[CADisplayLink displayLinkWithTarget:_displayLinkTarget
226 selector:@selector(displayLinkDidFire:)] retain];
227 [_displayLink setPreferredFramesPerSecond:60];
228 [_displayLink setPaused:YES];
229 [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSRunLoopCommonModes];
230}
231
232- (void)configureDesktopTexture
233{
234 [_desktopTexture release];
235 _desktopTexture = nil;
236
237 CGContextRef bitmapContext = (_session != nil) ? [_session bitmapContext] : nil;
238 if (!_metalDevice || !bitmapContext)
239 return;
240
241 NSUInteger width = CGBitmapContextGetWidth(bitmapContext);
242 NSUInteger height = CGBitmapContextGetHeight(bitmapContext);
243 if ((width == 0) || (height == 0))
244 return;
245
246 MTLTextureDescriptor *descriptor =
247 [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm
248 width:width
249 height:height
250 mipmapped:NO];
251 [descriptor setUsage:MTLTextureUsageShaderRead];
252 [descriptor setStorageMode:MTLStorageModeShared];
253 _desktopTexture = [_metalDevice newTextureWithDescriptor:descriptor];
254}
255
256- (void)prepareForBitmapContextChange
257{
258 [_displayLink setPaused:YES];
259 _hasPendingDirtyRect = NO;
260 _pendingDirtyRect = CGRectNull;
261 [_desktopTexture release];
262 _desktopTexture = nil;
263}
264
265- (void)setNeedsDisplayInRemoteRect:(CGRect)rect
266{
267 if (!_desktopTexture || CGRectIsNull(rect) || CGRectIsEmpty(rect))
268 return;
269
270 CGRect textureBounds = CGRectMake(0.0, 0.0, [_desktopTexture width], [_desktopTexture height]);
271 rect = CGRectIntersection(CGRectIntegral(rect), textureBounds);
272 if (CGRectIsNull(rect) || CGRectIsEmpty(rect))
273 return;
274
275 _pendingDirtyRect = _hasPendingDirtyRect ? CGRectUnion(_pendingDirtyRect, rect) : rect;
276 _hasPendingDirtyRect = YES;
277 [self scheduleRender];
278}
279
280- (void)scheduleRender
281{
282 if ([self window] && _displayLink && _hasPendingDirtyRect)
283 [_displayLink setPaused:NO];
284}
285
286- (void)didMoveToWindow
287{
288 [super didMoveToWindow];
289 [self scheduleRender];
290}
291
292- (void)layoutSubviews
293{
294 [super layoutSubviews];
295 CAMetalLayer *metalLayer = (CAMetalLayer *)[self layer];
296 CGSize size = [self bounds].size;
297 CGSize drawableSize = CGSizeMake(MAX(1.0, size.width), MAX(1.0, size.height));
298 if (CGSizeEqualToSize(drawableSize, _drawableSize))
299 return;
300
301 _drawableSize = drawableSize;
302 [metalLayer setDrawableSize:drawableSize];
303
304 if (_desktopTexture)
305 [self setNeedsDisplayInRemoteRect:CGRectMake(0, 0, [_desktopTexture width],
306 [_desktopTexture height])];
307}
308
309- (void)renderDisplayLink:(CADisplayLink *)displayLink
310{
311 (void)displayLink;
312 if (!_hasPendingDirtyRect || !_desktopTexture || !_pipelineState)
313 {
314 [_displayLink setPaused:YES];
315 return;
316 }
317
318 CGContextRef bitmapContext = (_session != nil) ? [_session bitmapContext] : nil;
319 BYTE *source = bitmapContext ? CGBitmapContextGetData(bitmapContext) : nullptr;
320 if (!source)
321 {
322 [_displayLink setPaused:YES];
323 return;
324 }
325
326 CGRect dirtyRect = _pendingDirtyRect;
327 _pendingDirtyRect = CGRectNull;
328 _hasPendingDirtyRect = NO;
329 NSUInteger x = (NSUInteger)CGRectGetMinX(dirtyRect);
330 NSUInteger y = (NSUInteger)CGRectGetMinY(dirtyRect);
331 NSUInteger width = (NSUInteger)CGRectGetWidth(dirtyRect);
332 NSUInteger height = (NSUInteger)CGRectGetHeight(dirtyRect);
333 NSUInteger bytesPerRow = CGBitmapContextGetBytesPerRow(bitmapContext);
334 const NSUInteger bytesPerPixel = 4;
335 MTLRegion region = MTLRegionMake2D(x, y, width, height);
336 [_desktopTexture replaceRegion:region
337 mipmapLevel:0
338 withBytes:source + (y * bytesPerRow) + (x * bytesPerPixel)
339 bytesPerRow:bytesPerRow];
340
341 id<CAMetalDrawable> drawable = [(CAMetalLayer *)[self layer] nextDrawable];
342 if (!drawable)
343 {
344 _pendingDirtyRect = dirtyRect;
345 _hasPendingDirtyRect = YES;
346 [_displayLink setPaused:YES];
347 return;
348 }
349
350 MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor];
351 [[pass colorAttachments][0] setTexture:[drawable texture]];
352 [[pass colorAttachments][0] setLoadAction:MTLLoadActionClear];
353 [[pass colorAttachments][0] setStoreAction:MTLStoreActionStore];
354 [[pass colorAttachments][0] setClearColor:MTLClearColorMake(0.0, 0.0, 0.0, 1.0)];
355
356 id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
357 id<MTLRenderCommandEncoder> encoder = [commandBuffer renderCommandEncoderWithDescriptor:pass];
358 [encoder setRenderPipelineState:_pipelineState];
359 [encoder setFragmentTexture:_desktopTexture atIndex:0];
360 [encoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
361 [encoder endEncoding];
362 [commandBuffer presentDrawable:drawable];
363 [commandBuffer commit];
364
365 if (!_hasPendingDirtyRect)
366 [_displayLink setPaused:YES];
367}
368
369static int VKFromHIDUsage(UIKeyboardHIDUsage usage)
370{
371 if (usage >= UIKeyboardHIDUsageKeyboardA && usage <= UIKeyboardHIDUsageKeyboardZ)
372 return VK_KEY_A + (usage - UIKeyboardHIDUsageKeyboardA);
373 if (usage >= UIKeyboardHIDUsageKeyboard1 && usage <= UIKeyboardHIDUsageKeyboard9)
374 return VK_KEY_1 + (usage - UIKeyboardHIDUsageKeyboard1);
375 if (usage >= UIKeyboardHIDUsageKeyboardF1 && usage <= UIKeyboardHIDUsageKeyboardF12)
376 return VK_F1 + (usage - UIKeyboardHIDUsageKeyboardF1);
377
378 switch (usage)
379 {
380 case UIKeyboardHIDUsageKeyboard0:
381 return VK_KEY_0;
382 case UIKeyboardHIDUsageKeyboardReturnOrEnter:
383 return VK_RETURN;
384 case UIKeyboardHIDUsageKeyboardEscape:
385 return VK_ESCAPE;
386 case UIKeyboardHIDUsageKeyboardDeleteOrBackspace:
387 return VK_BACK;
388 case UIKeyboardHIDUsageKeyboardTab:
389 return VK_TAB;
390 case UIKeyboardHIDUsageKeyboardSpacebar:
391 return VK_SPACE;
392 case UIKeyboardHIDUsageKeyboardHyphen:
393 return VK_OEM_MINUS;
394 case UIKeyboardHIDUsageKeyboardEqualSign:
395 return VK_OEM_PLUS;
396 case UIKeyboardHIDUsageKeyboardOpenBracket:
397 return VK_OEM_4;
398 case UIKeyboardHIDUsageKeyboardCloseBracket:
399 return VK_OEM_6;
400 case UIKeyboardHIDUsageKeyboardBackslash:
401 return VK_OEM_5;
402 case UIKeyboardHIDUsageKeyboardSemicolon:
403 return VK_OEM_1;
404 case UIKeyboardHIDUsageKeyboardQuote:
405 return VK_OEM_7;
406 case UIKeyboardHIDUsageKeyboardGraveAccentAndTilde:
407 return VK_OEM_3;
408 case UIKeyboardHIDUsageKeyboardComma:
409 return VK_OEM_COMMA;
410 case UIKeyboardHIDUsageKeyboardPeriod:
411 return VK_OEM_PERIOD;
412 case UIKeyboardHIDUsageKeyboardSlash:
413 return VK_OEM_2;
414 case UIKeyboardHIDUsageKeyboardCapsLock:
415 return VK_CAPITAL;
416 case UIKeyboardHIDUsageKeyboardRightArrow:
417 return VK_RIGHT | KBDEXT;
418 case UIKeyboardHIDUsageKeyboardLeftArrow:
419 return VK_LEFT | KBDEXT;
420 case UIKeyboardHIDUsageKeyboardDownArrow:
421 return VK_DOWN | KBDEXT;
422 case UIKeyboardHIDUsageKeyboardUpArrow:
423 return VK_UP | KBDEXT;
424 case UIKeyboardHIDUsageKeyboardInsert:
425 return VK_INSERT | KBDEXT;
426 case UIKeyboardHIDUsageKeyboardHome:
427 return VK_HOME | KBDEXT;
428 case UIKeyboardHIDUsageKeyboardPageUp:
429 return VK_PRIOR | KBDEXT;
430 case UIKeyboardHIDUsageKeyboardDeleteForward:
431 return VK_DELETE | KBDEXT;
432 case UIKeyboardHIDUsageKeyboardEnd:
433 return VK_END | KBDEXT;
434 case UIKeyboardHIDUsageKeyboardPageDown:
435 return VK_NEXT | KBDEXT;
436 case UIKeyboardHIDUsageKeyboardLeftControl:
437 return VK_LCONTROL;
438 case UIKeyboardHIDUsageKeyboardLeftShift:
439 return VK_LSHIFT;
440 case UIKeyboardHIDUsageKeyboardLeftAlt:
441 return VK_LMENU;
442 case UIKeyboardHIDUsageKeyboardLeftGUI:
443 return VK_LWIN | KBDEXT;
444 case UIKeyboardHIDUsageKeyboardRightControl:
445 return VK_RCONTROL | KBDEXT;
446 case UIKeyboardHIDUsageKeyboardRightShift:
447 return VK_RSHIFT;
448 case UIKeyboardHIDUsageKeyboardRightAlt:
449 return VK_RMENU | KBDEXT;
450 case UIKeyboardHIDUsageKeyboardRightGUI:
451 return VK_RWIN | KBDEXT;
452 default:
453 return 0;
454 }
455}
456
457- (BOOL)canBecomeFirstResponder
458{
459 return _hardwareKeyboardActive;
460}
461
462- (void)handlePresses:(NSSet<UIPress *> *)presses up:(BOOL)up
463{
464 for (UIPress *press in presses)
465 {
466 UIKey *key = press.key;
467 if (key == nil)
468 continue;
469
470 int vk = VKFromHIDUsage(key.keyCode);
471 if (vk != 0)
472 [[RDPKeyboard getSharedRDPKeyboard] sendVirtualKey:vk up:up];
473 }
474}
475
476- (void)pressesBegan:(NSSet<UIPress *> *)presses withEvent:(UIPressesEvent *)event
477{
478 if (_hardwareKeyboardActive)
479 [self handlePresses:presses up:NO];
480 else
481 [super pressesBegan:presses withEvent:event];
482}
483
484- (void)pressesEnded:(NSSet<UIPress *> *)presses withEvent:(UIPressesEvent *)event
485{
486 if (_hardwareKeyboardActive)
487 [self handlePresses:presses up:YES];
488 else
489 [super pressesEnded:presses withEvent:event];
490}
491
492- (void)dealloc
493{
494 [_displayLinkTarget clearView];
495 [_displayLink invalidate];
496 [_displayLink release];
497 [_displayLinkTarget release];
498 [_desktopTexture release];
499 [_pipelineState release];
500 [_commandQueue release];
501 [_metalDevice release];
502 [_cursorImageView release];
503 [super dealloc];
504}
505
506@end