Flutter Windows Embedder
accessibility_bridge_windows_unittests.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
7 #include <comdef.h>
8 #include <comutil.h>
9 #include <oleacc.h>
10 
11 #include <vector>
12 
13 #include "flutter/fml/macros.h"
14 #include "flutter/shell/platform/embedder/embedder.h"
15 #include "flutter/shell/platform/embedder/test_utils/proc_table_replacement.h"
19 #include "flutter/shell/platform/windows/testing/engine_modifier.h"
20 #include "flutter/shell/platform/windows/testing/mock_window_binding_handler.h"
21 #include "flutter/shell/platform/windows/testing/test_keyboard.h"
22 #include "gmock/gmock.h"
23 #include "gtest/gtest.h"
24 
25 namespace flutter {
26 namespace testing {
27 
28 namespace {
29 
30 // A structure representing a Win32 MSAA event targeting a specified node.
31 struct MsaaEvent {
32  std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate;
33  ax::mojom::Event event_type;
34 };
35 
36 // Accessibility bridge delegate that captures events dispatched to the OS.
37 class AccessibilityBridgeWindowsSpy : public AccessibilityBridgeWindows {
38  public:
40 
41  explicit AccessibilityBridgeWindowsSpy(FlutterWindowsEngine* engine,
42  FlutterWindowsView* view)
43  : AccessibilityBridgeWindows(view) {}
44 
45  void DispatchWinAccessibilityEvent(
46  std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate,
47  ax::mojom::Event event_type) override {
48  dispatched_events_.push_back({node_delegate, event_type});
49  }
50 
51  void SetFocus(std::shared_ptr<FlutterPlatformNodeDelegateWindows>
52  node_delegate) override {
53  focused_nodes_.push_back(std::move(node_delegate));
54  }
55 
56  void ResetRecords() {
57  dispatched_events_.clear();
58  focused_nodes_.clear();
59  }
60 
61  const std::vector<MsaaEvent>& dispatched_events() const {
62  return dispatched_events_;
63  }
64 
65  const std::vector<int32_t> focused_nodes() const {
66  std::vector<int32_t> ids;
67  std::transform(focused_nodes_.begin(), focused_nodes_.end(),
68  std::back_inserter(ids),
69  [](std::shared_ptr<FlutterPlatformNodeDelegate> node) {
70  return node->GetAXNode()->id();
71  });
72  return ids;
73  }
74 
75  protected:
76  std::weak_ptr<FlutterPlatformNodeDelegate> GetFocusedNode() override {
77  return focused_nodes_.back();
78  }
79 
80  private:
81  std::vector<MsaaEvent> dispatched_events_;
82  std::vector<std::shared_ptr<FlutterPlatformNodeDelegate>> focused_nodes_;
83 
84  FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridgeWindowsSpy);
85 };
86 
87 // A FlutterWindowsView whose accessibility bridge is an
88 // AccessibilityBridgeWindowsSpy.
89 class FlutterWindowsViewSpy : public FlutterWindowsView {
90  public:
91  explicit FlutterWindowsViewSpy(std::unique_ptr<WindowBindingHandler> handler)
92  : FlutterWindowsView(std::move(handler)) {}
93 
94  protected:
95  virtual std::shared_ptr<AccessibilityBridgeWindows>
96  CreateAccessibilityBridge() override {
97  return std::make_shared<AccessibilityBridgeWindowsSpy>(GetEngine(), this);
98  }
99 
100  private:
101  FML_DISALLOW_COPY_AND_ASSIGN(FlutterWindowsViewSpy);
102 };
103 
104 // Returns an engine instance configured with dummy project path values, and
105 // overridden methods for sending platform messages, so that the engine can
106 // respond as if the framework were connected.
107 std::unique_ptr<FlutterWindowsEngine> GetTestEngine() {
108  FlutterDesktopEngineProperties properties = {};
109  properties.assets_path = L"C:\\foo\\flutter_assets";
110  properties.icu_data_path = L"C:\\foo\\icudtl.dat";
111  properties.aot_library_path = L"C:\\foo\\aot.so";
112  FlutterProjectBundle project(properties);
113  auto engine = std::make_unique<FlutterWindowsEngine>(project);
114 
115  EngineModifier modifier(engine.get());
116  modifier.embedder_api().UpdateSemanticsEnabled =
117  [](FLUTTER_API_SYMBOL(FlutterEngine) engine, bool enabled) {
118  return kSuccess;
119  };
120 
121  MockEmbedderApiForKeyboard(modifier,
122  std::make_shared<MockKeyResponseController>());
123 
124  engine->Run();
125  return engine;
126 }
127 
128 // Populates the AXTree associated with the specified bridge with test data.
129 //
130 // node0
131 // / \
132 // node1 node2
133 // / \
134 // node3 node4
135 //
136 // node0 and node2 are grouping nodes. node1 and node2 are static text nodes.
137 // node4 is a static text node with no text, and hence has the "ignored" state.
138 void PopulateAXTree(std::shared_ptr<AccessibilityBridge> bridge) {
139  // Add node 0: root.
140  FlutterSemanticsNode2 node0{sizeof(FlutterSemanticsNode2), 0};
141  std::vector<int32_t> node0_children{1, 2};
142  node0.child_count = node0_children.size();
143  node0.children_in_traversal_order = node0_children.data();
144  node0.children_in_hit_test_order = node0_children.data();
145 
146  // Add node 1: text child of node 0.
147  FlutterSemanticsNode2 node1{sizeof(FlutterSemanticsNode2), 1};
148  node1.label = "prefecture";
149  node1.value = "Kyoto";
150 
151  // Add node 2: subtree child of node 0.
152  FlutterSemanticsNode2 node2{sizeof(FlutterSemanticsNode2), 2};
153  std::vector<int32_t> node2_children{3, 4};
154  node2.child_count = node2_children.size();
155  node2.children_in_traversal_order = node2_children.data();
156  node2.children_in_hit_test_order = node2_children.data();
157 
158  // Add node 3: text child of node 2.
159  FlutterSemanticsNode2 node3{sizeof(FlutterSemanticsNode2), 3};
160  node3.label = "city";
161  node3.value = "Uji";
162 
163  // Add node 4: text child (with no text) of node 2.
164  FlutterSemanticsNode2 node4{sizeof(FlutterSemanticsNode2), 4};
165 
166  bridge->AddFlutterSemanticsNodeUpdate(node0);
167  bridge->AddFlutterSemanticsNodeUpdate(node1);
168  bridge->AddFlutterSemanticsNodeUpdate(node2);
169  bridge->AddFlutterSemanticsNodeUpdate(node3);
170  bridge->AddFlutterSemanticsNodeUpdate(node4);
171  bridge->CommitUpdates();
172 }
173 
174 ui::AXNode* AXNodeFromID(std::shared_ptr<AccessibilityBridge> bridge,
175  int32_t id) {
176  auto node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(id).lock();
177  return node_delegate ? node_delegate->GetAXNode() : nullptr;
178 }
179 
180 std::shared_ptr<AccessibilityBridgeWindowsSpy> GetAccessibilityBridgeSpy(
181  FlutterWindowsView& view) {
182  return std::static_pointer_cast<AccessibilityBridgeWindowsSpy>(
183  view.accessibility_bridge().lock());
184 }
185 
186 void ExpectWinEventFromAXEvent(int32_t node_id,
187  ui::AXEventGenerator::Event ax_event,
188  ax::mojom::Event expected_event) {
189  auto engine = GetTestEngine();
190  auto window_binding_handler =
191  std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
192  FlutterWindowsViewSpy view(std::move(window_binding_handler));
193  view.SetEngine(engine.get());
194  view.OnUpdateSemanticsEnabled(true);
195 
196  auto bridge = GetAccessibilityBridgeSpy(view);
197  PopulateAXTree(bridge);
198 
199  bridge->ResetRecords();
200  bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
201  {ax_event, ax::mojom::EventFrom::kNone, {}}});
202  ASSERT_EQ(bridge->dispatched_events().size(), 1);
203  EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
204 }
205 
206 void ExpectWinEventFromAXEventOnFocusNode(int32_t node_id,
207  ui::AXEventGenerator::Event ax_event,
208  ax::mojom::Event expected_event,
209  int32_t focus_id) {
210  auto engine = GetTestEngine();
211  auto window_binding_handler =
212  std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
213  FlutterWindowsViewSpy view(std::move(window_binding_handler));
214  view.SetEngine(engine.get());
215  view.OnUpdateSemanticsEnabled(true);
216 
217  auto bridge = GetAccessibilityBridgeSpy(view);
218  PopulateAXTree(bridge);
219 
220  bridge->ResetRecords();
221  auto focus_delegate =
222  bridge->GetFlutterPlatformNodeDelegateFromID(focus_id).lock();
223  bridge->SetFocus(std::static_pointer_cast<FlutterPlatformNodeDelegateWindows>(
224  focus_delegate));
225  bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
226  {ax_event, ax::mojom::EventFrom::kNone, {}}});
227  ASSERT_EQ(bridge->dispatched_events().size(), 1);
228  EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
229  EXPECT_EQ(bridge->dispatched_events()[0].node_delegate->GetAXNode()->id(),
230  focus_id);
231 }
232 
233 } // namespace
234 
236  auto engine = GetTestEngine();
237  auto window_binding_handler =
238  std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
239  FlutterWindowsViewSpy view(std::move(window_binding_handler));
240  view.SetEngine(engine.get());
241  view.OnUpdateSemanticsEnabled(true);
242 
243  auto bridge = view.accessibility_bridge().lock();
244  PopulateAXTree(bridge);
245 
246  auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
247  auto node1_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock();
248  EXPECT_EQ(node0_delegate->GetNativeViewAccessible(),
249  node1_delegate->GetParent());
250 }
251 
252 TEST(AccessibilityBridgeWindows, GetParentOnRootRetunsNullptr) {
253  auto engine = GetTestEngine();
254  auto window_binding_handler =
255  std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
256  FlutterWindowsViewSpy view(std::move(window_binding_handler));
257  view.SetEngine(engine.get());
258  view.OnUpdateSemanticsEnabled(true);
259 
260  auto bridge = view.accessibility_bridge().lock();
261  PopulateAXTree(bridge);
262 
263  auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
264  ASSERT_TRUE(node0_delegate->GetParent() == nullptr);
265 }
266 
267 TEST(AccessibilityBridgeWindows, DispatchAccessibilityAction) {
268  auto engine = GetTestEngine();
269  auto window_binding_handler =
270  std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
271  FlutterWindowsViewSpy view(std::move(window_binding_handler));
272  view.SetEngine(engine.get());
273  view.OnUpdateSemanticsEnabled(true);
274 
275  auto bridge = view.accessibility_bridge().lock();
276  PopulateAXTree(bridge);
277 
278  FlutterSemanticsAction actual_action = kFlutterSemanticsActionTap;
279  EngineModifier modifier(view.GetEngine());
280  modifier.embedder_api().DispatchSemanticsAction = MOCK_ENGINE_PROC(
281  DispatchSemanticsAction,
282  ([&actual_action](FLUTTER_API_SYMBOL(FlutterEngine) engine, uint64_t id,
283  FlutterSemanticsAction action, const uint8_t* data,
284  size_t data_length) {
285  actual_action = action;
286  return kSuccess;
287  }));
288 
289  AccessibilityBridgeWindows delegate(&view);
290  delegate.DispatchAccessibilityAction(1, kFlutterSemanticsActionCopy, {});
291  EXPECT_EQ(actual_action, kFlutterSemanticsActionCopy);
292 }
293 
294 TEST(AccessibilityBridgeWindows, OnAccessibilityEventAlert) {
295  ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::ALERT,
296  ax::mojom::Event::kAlert);
297 }
298 
299 TEST(AccessibilityBridgeWindows, OnAccessibilityEventChildrenChanged) {
300  ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::CHILDREN_CHANGED,
301  ax::mojom::Event::kChildrenChanged);
302 }
303 
304 TEST(AccessibilityBridgeWindows, OnAccessibilityEventFocusChanged) {
305  auto engine = GetTestEngine();
306  auto window_binding_handler =
307  std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
308  FlutterWindowsViewSpy view(std::move(window_binding_handler));
309  view.SetEngine(engine.get());
310  view.OnUpdateSemanticsEnabled(true);
311 
312  auto bridge = GetAccessibilityBridgeSpy(view);
313  PopulateAXTree(bridge);
314 
315  bridge->ResetRecords();
316  bridge->OnAccessibilityEvent({AXNodeFromID(bridge, 1),
317  {ui::AXEventGenerator::Event::FOCUS_CHANGED,
318  ax::mojom::EventFrom::kNone,
319  {}}});
320  ASSERT_EQ(bridge->dispatched_events().size(), 1);
321  EXPECT_EQ(bridge->dispatched_events()[0].event_type,
322  ax::mojom::Event::kFocus);
323 
324  ASSERT_EQ(bridge->focused_nodes().size(), 1);
325  EXPECT_EQ(bridge->focused_nodes()[0], 1);
326 }
327 
328 TEST(AccessibilityBridgeWindows, OnAccessibilityEventIgnoredChanged) {
329  // Static test nodes with no text, hint, or scrollability are ignored.
330  ExpectWinEventFromAXEvent(4, ui::AXEventGenerator::Event::IGNORED_CHANGED,
331  ax::mojom::Event::kHide);
332 }
333 
334 TEST(AccessibilityBridgeWindows, OnAccessibilityImageAnnotationChanged) {
335  ExpectWinEventFromAXEvent(
336  1, ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED,
337  ax::mojom::Event::kTextChanged);
338 }
339 
340 TEST(AccessibilityBridgeWindows, OnAccessibilityLiveRegionChanged) {
341  ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::LIVE_REGION_CHANGED,
342  ax::mojom::Event::kLiveRegionChanged);
343 }
344 
345 TEST(AccessibilityBridgeWindows, OnAccessibilityNameChanged) {
346  ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::NAME_CHANGED,
347  ax::mojom::Event::kTextChanged);
348 }
349 
350 TEST(AccessibilityBridgeWindows, OnAccessibilityHScrollPosChanged) {
351  ExpectWinEventFromAXEvent(
352  1, ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED,
353  ax::mojom::Event::kScrollPositionChanged);
354 }
355 
356 TEST(AccessibilityBridgeWindows, OnAccessibilityVScrollPosChanged) {
357  ExpectWinEventFromAXEvent(
358  1, ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED,
359  ax::mojom::Event::kScrollPositionChanged);
360 }
361 
362 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChanged) {
363  ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::SELECTED_CHANGED,
364  ax::mojom::Event::kValueChanged);
365 }
366 
367 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChildrenChanged) {
368  ExpectWinEventFromAXEvent(
369  2, ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED,
370  ax::mojom::Event::kSelectedChildrenChanged);
371 }
372 
373 TEST(AccessibilityBridgeWindows, OnAccessibilitySubtreeCreated) {
374  ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::SUBTREE_CREATED,
375  ax::mojom::Event::kShow);
376 }
377 
378 TEST(AccessibilityBridgeWindows, OnAccessibilityValueChanged) {
379  ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::VALUE_CHANGED,
380  ax::mojom::Event::kValueChanged);
381 }
382 
383 TEST(AccessibilityBridgeWindows, OnAccessibilityStateChanged) {
384  ExpectWinEventFromAXEvent(
385  1, ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED,
386  ax::mojom::Event::kStateChanged);
387 }
388 
389 TEST(AccessibilityBridgeWindows, OnDocumentSelectionChanged) {
390  ExpectWinEventFromAXEventOnFocusNode(
391  1, ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED,
392  ax::mojom::Event::kDocumentSelectionChanged, 2);
393 }
394 
395 } // namespace testing
396 } // namespace flutter
FlutterDesktopEngineProperties::aot_library_path
const wchar_t * aot_library_path
Definition: flutter_windows.h:51
FlutterDesktopEngineProperties
Definition: flutter_windows.h:36
flutter::FlutterEngine
Definition: flutter_engine.h:28
flutter::AccessibilityBridgeWindows::OnAccessibilityEvent
void OnAccessibilityEvent(ui::AXEventGenerator::TargetedEvent targeted_event) override
Handle accessibility events generated due to accessibility tree changes. These events are needed to b...
Definition: accessibility_bridge_windows.cc:18
flutter::AccessibilityBridgeWindows
Definition: accessibility_bridge_windows.h:27
FlutterDesktopEngineProperties::icu_data_path
const wchar_t * icu_data_path
Definition: flutter_windows.h:45
flutter_windows_view.h
accessibility_bridge_windows.h
flutter::AccessibilityBridgeWindows::DispatchAccessibilityAction
void DispatchAccessibilityAction(AccessibilityNodeId target, FlutterSemanticsAction action, fml::MallocMapping data) override
Dispatch accessibility action back to the Flutter framework. These actions are generated in the nativ...
Definition: accessibility_bridge_windows.cc:162
flutter
Definition: accessibility_bridge_windows.cc:11
flutter_windows_engine.h
flutter_platform_node_delegate_windows.h
flutter::testing::TEST
TEST(AccessibilityBridgeWindows, GetParent)
Definition: accessibility_bridge_windows_unittests.cc:235
action
int action
Definition: keyboard_key_handler_unittests.cc:116
event_type
ax::mojom::Event event_type
Definition: accessibility_bridge_windows_unittests.cc:33
FlutterDesktopEngineProperties::assets_path
const wchar_t * assets_path
Definition: flutter_windows.h:40
node_delegate
std::shared_ptr< FlutterPlatformNodeDelegateWindows > node_delegate
Definition: accessibility_bridge_windows_unittests.cc:32