Flutter iOS Embedder
FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include <IOSurface/IOSurfaceObjC.h>
6 #include <Metal/Metal.h>
7 #include <UIKit/UIKit.h>
8 
9 #include "flutter/fml/logging.h"
11 
12 @interface DisplayLinkManager : NSObject
13 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
14 + (double)displayRefreshRate;
15 @end
16 
17 @class FlutterTexture;
18 @class FlutterDrawable;
19 
20 extern CFTimeInterval display_link_target;
21 
22 @interface FlutterMetalLayer () {
23  id<MTLDevice> _preferredDevice;
24  CGSize _drawableSize;
25 
26  NSUInteger _nextDrawableId;
27 
28  NSMutableSet<FlutterTexture*>* _availableTextures;
29  NSUInteger _totalTextures;
30 
32 
33  // There must be a CADisplayLink scheduled *on main thread* otherwise
34  // core animation only updates layers 60 times a second.
35  CADisplayLink* _displayLink;
37 
38  // Used to track whether the content was set during this display link.
39  // When unlocking phone the layer (main thread) display link and raster thread
40  // display link get out of sync for several seconds. Even worse, layer display
41  // link does not seem to reflect actual vsync. Forcing the layer link
42  // to max rate (instead range) temporarily seems to fix the issue.
44 
45  // Whether layer displayLink is forced to max rate.
47 }
48 
49 - (void)presentTexture:(FlutterTexture*)texture;
50 - (void)returnTexture:(FlutterTexture*)texture;
51 
52 @end
53 
54 @interface FlutterTexture : NSObject {
55  id<MTLTexture> _texture;
56  IOSurface* _surface;
57  CFTimeInterval _presentedTime;
58 }
59 
60 @property(readonly, nonatomic) id<MTLTexture> texture;
61 @property(readonly, nonatomic) IOSurface* surface;
62 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
63 
64 @end
65 
66 @implementation FlutterTexture
67 
68 @synthesize texture = _texture;
69 @synthesize surface = _surface;
70 @synthesize presentedTime = _presentedTime;
71 
72 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
73  if (self = [super init]) {
74  _texture = texture;
75  _surface = surface;
76  }
77  return self;
78 }
79 
80 @end
81 
82 @interface FlutterDrawable : NSObject <CAMetalDrawable> {
85  NSUInteger _drawableId;
86  BOOL _presented;
87 }
88 
89 - (instancetype)initWithTexture:(FlutterTexture*)texture
90  layer:(FlutterMetalLayer*)layer
91  drawableId:(NSUInteger)drawableId;
92 
93 @end
94 
95 @implementation FlutterDrawable
96 
97 - (instancetype)initWithTexture:(FlutterTexture*)texture
98  layer:(FlutterMetalLayer*)layer
99  drawableId:(NSUInteger)drawableId {
100  if (self = [super init]) {
101  _texture = texture;
102  _layer = layer;
103  _drawableId = drawableId;
104  }
105  return self;
106 }
107 
108 - (id<MTLTexture>)texture {
109  return self->_texture.texture;
110 }
111 
112 #pragma clang diagnostic push
113 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
114 - (CAMetalLayer*)layer {
115  return (id)self->_layer;
116 }
117 #pragma clang diagnostic pop
118 
119 - (NSUInteger)drawableID {
120  return self->_drawableId;
121 }
122 
123 - (CFTimeInterval)presentedTime {
124  return 0;
125 }
126 
127 - (void)present {
128  [_layer presentTexture:self->_texture];
129  self->_presented = YES;
130 }
131 
132 - (void)dealloc {
133  if (!_presented) {
134  [_layer returnTexture:self->_texture];
135  }
136 }
137 
138 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
139  FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement addPresentedHandler:";
140 }
141 
142 - (void)presentAtTime:(CFTimeInterval)presentationTime {
143  FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAtTime:";
144 }
145 
146 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
147  FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:";
148 }
149 
150 @end
151 
152 @implementation FlutterMetalLayer
153 
154 @synthesize preferredDevice = _preferredDevice;
155 @synthesize device = _device;
156 @synthesize pixelFormat = _pixelFormat;
157 @synthesize framebufferOnly = _framebufferOnly;
158 @synthesize colorspace = _colorspace;
159 @synthesize wantsExtendedDynamicRangeContent = _wantsExtendedDynamicRangeContent;
160 
161 - (instancetype)init {
162  if (self = [super init]) {
163  _preferredDevice = MTLCreateSystemDefaultDevice();
164  self.device = self.preferredDevice;
165  self.pixelFormat = MTLPixelFormatBGRA8Unorm;
166  _availableTextures = [[NSMutableSet alloc] init];
167 
168  _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(onDisplayLink:)];
169  [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:NO];
170  [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
171  [[NSNotificationCenter defaultCenter] addObserver:self
172  selector:@selector(didEnterBackground:)
173  name:UIApplicationDidEnterBackgroundNotification
174  object:nil];
175  }
176  return self;
177 }
178 
179 - (void)dealloc {
180  [[NSNotificationCenter defaultCenter] removeObserver:self];
181 }
182 
183 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
184  // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
185  // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
186  // has it's own displaylink scheduled on main thread, which is used to trigger core animation
187  // frame allowing for 120hz updates.
189  return;
190  }
191  double maxFrameRate = fmax(refreshRate, 60);
192  double minFrameRate = fmax(maxFrameRate / 2, 60);
193  if (@available(iOS 15.0, *)) {
194  _displayLink.preferredFrameRateRange =
195  CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
196  } else {
197  _displayLink.preferredFramesPerSecond = maxFrameRate;
198  }
199 }
200 
201 - (void)onDisplayLink:(CADisplayLink*)link {
202  _didSetContentsDuringThisDisplayLinkPeriod = NO;
203  // Do not pause immediately, this seems to prevent 120hz while touching.
204  if (_displayLinkPauseCountdown == 3) {
205  _displayLink.paused = YES;
206  if (_displayLinkForcedMaxRate) {
207  [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:NO];
208  _displayLinkForcedMaxRate = NO;
209  }
210  } else {
211  ++_displayLinkPauseCountdown;
212  }
213 }
214 
215 - (BOOL)isKindOfClass:(Class)aClass {
216 #pragma clang diagnostic push
217 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
218  // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
219  if ([aClass isEqual:[CAMetalLayer class]]) {
220  return YES;
221  }
222 #pragma clang diagnostic pop
223  return [super isKindOfClass:aClass];
224 }
225 
226 - (void)setDrawableSize:(CGSize)drawableSize {
227  [_availableTextures removeAllObjects];
228  _front = nil;
229  _totalTextures = 0;
230  _drawableSize = drawableSize;
231 }
232 
233 - (void)didEnterBackground:(id)notification {
234  [_availableTextures removeAllObjects];
235  _totalTextures = _front != nil ? 1 : 0;
236  _displayLink.paused = YES;
237 }
238 
239 - (CGSize)drawableSize {
240  return _drawableSize;
241 }
242 
243 - (IOSurface*)createIOSurface {
244  unsigned pixelFormat;
245  unsigned bytesPerElement;
246  if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
247  pixelFormat = kCVPixelFormatType_64RGBAHalf;
248  bytesPerElement = 8;
249  } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
250  pixelFormat = kCVPixelFormatType_32BGRA;
251  bytesPerElement = 4;
252  } else {
253  FML_LOG(ERROR) << "Unsupported pixel format: " << self.pixelFormat;
254  return nil;
255  }
256  size_t bytesPerRow =
257  IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
258  size_t totalBytes =
259  IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
260  NSDictionary* options = @{
261  (id)kIOSurfaceWidth : @(_drawableSize.width),
262  (id)kIOSurfaceHeight : @(_drawableSize.height),
263  (id)kIOSurfacePixelFormat : @(pixelFormat),
264  (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
265  (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
266  (id)kIOSurfaceAllocSize : @(totalBytes),
267  };
268 
269  IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
270  if (res == nil) {
271  FML_LOG(ERROR) << "Failed to create IOSurface with options "
272  << options.debugDescription.UTF8String;
273  return nil;
274  }
275 
276  if (self.colorspace != nil) {
277  CFStringRef name = CGColorSpaceGetName(self.colorspace);
278  IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), name);
279  } else {
280  IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), kCGColorSpaceSRGB);
281  }
282  return (__bridge_transfer IOSurface*)res;
283 }
284 
285 - (FlutterTexture*)nextTexture {
286  @synchronized(self) {
287  if (_totalTextures < 3) {
288  ++_totalTextures;
289  IOSurface* surface = [self createIOSurface];
290  if (surface == nil) {
291  return nil;
292  }
293  MTLTextureDescriptor* textureDescriptor =
294  [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
295  width:_drawableSize.width
296  height:_drawableSize.height
297  mipmapped:NO];
298 
299  if (_framebufferOnly) {
300  textureDescriptor.usage = MTLTextureUsageRenderTarget;
301  } else {
302  textureDescriptor.usage =
303  MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
304  }
305  id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
306  iosurface:(__bridge IOSurfaceRef)surface
307  plane:0];
308  FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
309  surface:surface];
310  return flutterTexture;
311  } else {
312  // Make sure raster thread doesn't have too many drawables in flight.
313  if (_availableTextures.count == 0) {
314  CFTimeInterval start = CACurrentMediaTime();
315  while (_availableTextures.count == 0 && CACurrentMediaTime() - start < 1.0) {
316  usleep(100);
317  }
318  CFTimeInterval elapsed = CACurrentMediaTime() - start;
319  if (_availableTextures.count == 0) {
320  NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
321  return nil;
322  } else {
323  NSLog(@"Had to wait %f seconds for a drawable", elapsed);
324  }
325  }
326 
327  // Prefer surface that is not in use and has been presented the longest
328  // time ago.
329  // When isInUse is false, the surface is definitely not used by the compositor.
330  // When isInUse is true, the surface may be used by the compositor.
331  // When both surfaces are in use, the one presented earlier will be returned.
332  // The assumption here is that the compositor is already aware of the
333  // newer texture and is unlikely to read from the older one, even though it
334  // has not decreased the use count yet (there seems to be certain latency).
335  FlutterTexture* res = nil;
336  for (FlutterTexture* texture in _availableTextures) {
337  if (res == nil) {
338  res = texture;
339  } else if (res.surface.isInUse && !texture.surface.isInUse) {
340  // prefer texture that is not in use.
341  res = texture;
342  } else if (res.surface.isInUse == texture.surface.isInUse &&
343  texture.presentedTime < res.presentedTime) {
344  // prefer texture with older presented time.
345  res = texture;
346  }
347  }
348  [_availableTextures removeObject:res];
349  return res;
350  }
351  }
352 }
353 
354 - (id<CAMetalDrawable>)nextDrawable {
355  FlutterTexture* texture = [self nextTexture];
356  if (texture == nil) {
357  return nil;
358  }
359  FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
360  layer:self
361  drawableId:_nextDrawableId++];
362  return drawable;
363 }
364 
365 - (void)presentOnMainThread:(FlutterTexture*)texture {
366  // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
367  // Might also be placebo
368  [self setNeedsDisplay];
369 
370  [CATransaction begin];
371  [CATransaction setDisableActions:YES];
372  self.contents = texture.surface;
373  texture.presentedTime = CACurrentMediaTime();
374  [CATransaction commit];
375  _displayLink.paused = NO;
376  _displayLinkPauseCountdown = 0;
377  if (!_didSetContentsDuringThisDisplayLinkPeriod) {
378  _didSetContentsDuringThisDisplayLinkPeriod = YES;
379  } else if (!_displayLinkForcedMaxRate) {
380  _displayLinkForcedMaxRate = YES;
381  [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:YES];
382  }
383 }
384 
385 - (void)presentTexture:(FlutterTexture*)texture {
386  @synchronized(self) {
387  if (_front != nil) {
388  [_availableTextures addObject:_front];
389  }
390  _front = texture;
391  if ([NSThread isMainThread]) {
392  [self presentOnMainThread:texture];
393  } else {
394  // Core animation layers can only be updated on main thread.
395  dispatch_async(dispatch_get_main_queue(), ^{
396  [self presentOnMainThread:texture];
397  });
398  }
399  }
400 }
401 
402 - (void)returnTexture:(FlutterTexture*)texture {
403  @synchronized(self) {
404  [_availableTextures addObject:texture];
405  }
406 }
407 
408 + (BOOL)enabled {
409  static BOOL enabled = NO;
410  static BOOL didCheckInfoPlist = NO;
411  if (!didCheckInfoPlist) {
412  didCheckInfoPlist = YES;
413  NSNumber* use_flutter_metal_layer =
414  [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
415  if (use_flutter_metal_layer != nil && [use_flutter_metal_layer boolValue]) {
416  enabled = YES;
417  FML_LOG(WARNING) << "Using FlutterMetalLayer. This is an experimental feature.";
418  }
419  }
420  return enabled;
421 }
422 
423 @end
FlutterMetalLayer::wantsExtendedDynamicRangeContent
BOOL wantsExtendedDynamicRangeContent
Definition: FlutterMetalLayer.h:18
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:408
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:83
FlutterTexture::_surface
IOSurface * _surface
Definition: FlutterMetalLayer.mm:56
FlutterDrawable
Definition: FlutterMetalLayer.mm:82
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:23
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:85
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:43
FlutterMetalLayer::framebufferOnly
BOOL framebufferOnly
Definition: FlutterMetalLayer.h:14
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:29
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:31
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:17
FlutterTexture::_presentedTime
CFTimeInterval _presentedTime
Definition: FlutterMetalLayer.mm:57
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:24
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:36
FlutterMetalLayer::preferredDevice
id< MTLDevice > preferredDevice
Definition: FlutterMetalLayer.h:12
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:28
FlutterTexture
Definition: FlutterMetalLayer.mm:54
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:26
FlutterTexture::_texture
id< MTLTexture > _texture
Definition: FlutterMetalLayer.mm:55
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:62
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:35
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:61
FlutterMetalLayer::device
id< MTLDevice > device
Definition: FlutterMetalLayer.h:11
FlutterMetalLayer.h
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:86
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:354
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:13
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:46
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:15
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:84
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:60
FlutterMetalLayer
Definition: FlutterMetalLayer.h:9