1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
|
//===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This family of functions perform manipulations on basic blocks, and
// instructions contained within basic blocks.
//
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryDependenceAnalysis.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <cstdint>
#include <string>
#include <utility>
#include <vector>
using namespace llvm;
#define DEBUG_TYPE "basicblock-utils"
static cl::opt<unsigned> MaxDeoptOrUnreachableSuccessorCheckDepth(
"max-deopt-or-unreachable-succ-check-depth", cl::init(8), cl::Hidden,
cl::desc("Set the maximum path length when checking whether a basic block "
"is followed by a block that either has a terminating "
"deoptimizing call or is terminated with an unreachable"));
void llvm::detachDeadBlocks(
ArrayRef<BasicBlock *> BBs,
SmallVectorImpl<DominatorTree::UpdateType> *Updates,
bool KeepOneInputPHIs) {
for (auto *BB : BBs) {
// Loop through all of our successors and make sure they know that one
// of their predecessors is going away.
SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
for (BasicBlock *Succ : successors(BB)) {
Succ->removePredecessor(BB, KeepOneInputPHIs);
if (Updates && UniqueSuccessors.insert(Succ).second)
Updates->push_back({DominatorTree::Delete, BB, Succ});
}
// Zap all the instructions in the block.
while (!BB->empty()) {
Instruction &I = BB->back();
// If this instruction is used, replace uses with an arbitrary value.
// Because control flow can't get here, we don't care what we replace the
// value with. Note that since this block is unreachable, and all values
// contained within it must dominate their uses, that all uses will
// eventually be removed (they are themselves dead).
if (!I.use_empty())
I.replaceAllUsesWith(PoisonValue::get(I.getType()));
BB->back().eraseFromParent();
}
new UnreachableInst(BB->getContext(), BB);
assert(BB->size() == 1 &&
isa<UnreachableInst>(BB->getTerminator()) &&
"The successor list of BB isn't empty before "
"applying corresponding DTU updates.");
}
}
void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
bool KeepOneInputPHIs) {
DeleteDeadBlocks({BB}, DTU, KeepOneInputPHIs);
}
void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
bool KeepOneInputPHIs) {
#ifndef NDEBUG
// Make sure that all predecessors of each dead block is also dead.
SmallPtrSet<BasicBlock *, 4> Dead(BBs.begin(), BBs.end());
assert(Dead.size() == BBs.size() && "Duplicating blocks?");
for (auto *BB : Dead)
for (BasicBlock *Pred : predecessors(BB))
assert(Dead.count(Pred) && "All predecessors must be dead!");
#endif
SmallVector<DominatorTree::UpdateType, 4> Updates;
detachDeadBlocks(BBs, DTU ? &Updates : nullptr, KeepOneInputPHIs);
if (DTU)
DTU->applyUpdates(Updates);
for (BasicBlock *BB : BBs)
if (DTU)
DTU->deleteBB(BB);
else
BB->eraseFromParent();
}
bool llvm::EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
bool KeepOneInputPHIs) {
df_iterator_default_set<BasicBlock*> Reachable;
// Mark all reachable blocks.
for (BasicBlock *BB : depth_first_ext(&F, Reachable))
(void)BB/* Mark all reachable blocks */;
// Collect all dead blocks.
std::vector<BasicBlock*> DeadBlocks;
for (BasicBlock &BB : F)
if (!Reachable.count(&BB))
DeadBlocks.push_back(&BB);
// Delete the dead blocks.
DeleteDeadBlocks(DeadBlocks, DTU, KeepOneInputPHIs);
return !DeadBlocks.empty();
}
bool llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
MemoryDependenceResults *MemDep) {
if (!isa<PHINode>(BB->begin()))
return false;
while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
if (PN->getIncomingValue(0) != PN)
PN->replaceAllUsesWith(PN->getIncomingValue(0));
else
PN->replaceAllUsesWith(PoisonValue::get(PN->getType()));
if (MemDep)
MemDep->removeInstruction(PN); // Memdep updates AA itself.
PN->eraseFromParent();
}
return true;
}
bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
MemorySSAUpdater *MSSAU) {
// Recursively deleting a PHI may cause multiple PHIs to be deleted
// or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
SmallVector<WeakTrackingVH, 8> PHIs;
for (PHINode &PN : BB->phis())
PHIs.push_back(&PN);
bool Changed = false;
for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
if (PHINode *PN = dyn_cast_or_null<PHINode>(PHIs[i].operator Value*()))
Changed |= RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU);
return Changed;
}
bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
LoopInfo *LI, MemorySSAUpdater *MSSAU,
MemoryDependenceResults *MemDep,
bool PredecessorWithTwoSuccessors,
DominatorTree *DT) {
if (BB->hasAddressTaken())
return false;
// Can't merge if there are multiple predecessors, or no predecessors.
BasicBlock *PredBB = BB->getUniquePredecessor();
if (!PredBB) return false;
// Don't break self-loops.
if (PredBB == BB) return false;
// Don't break unwinding instructions or terminators with other side-effects.
Instruction *PTI = PredBB->getTerminator();
if (PTI->isSpecialTerminator() || PTI->mayHaveSideEffects())
return false;
// Can't merge if there are multiple distinct successors.
if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
return false;
// Currently only allow PredBB to have two predecessors, one being BB.
// Update BI to branch to BB's only successor instead of BB.
BranchInst *PredBB_BI;
BasicBlock *NewSucc = nullptr;
unsigned FallThruPath;
if (PredecessorWithTwoSuccessors) {
if (!(PredBB_BI = dyn_cast<BranchInst>(PTI)))
return false;
BranchInst *BB_JmpI = dyn_cast<BranchInst>(BB->getTerminator());
if (!BB_JmpI || !BB_JmpI->isUnconditional())
return false;
NewSucc = BB_JmpI->getSuccessor(0);
FallThruPath = PredBB_BI->getSuccessor(0) == BB ? 0 : 1;
}
// Can't merge if there is PHI loop.
for (PHINode &PN : BB->phis())
if (llvm::is_contained(PN.incoming_values(), &PN))
return false;
LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
<< PredBB->getName() << "\n");
// Begin by getting rid of unneeded PHIs.
SmallVector<AssertingVH<Value>, 4> IncomingValues;
if (isa<PHINode>(BB->front())) {
for (PHINode &PN : BB->phis())
if (!isa<PHINode>(PN.getIncomingValue(0)) ||
cast<PHINode>(PN.getIncomingValue(0))->getParent() != BB)
IncomingValues.push_back(PN.getIncomingValue(0));
FoldSingleEntryPHINodes(BB, MemDep);
}
if (DT) {
assert(!DTU && "cannot use both DT and DTU for updates");
DomTreeNode *PredNode = DT->getNode(PredBB);
DomTreeNode *BBNode = DT->getNode(BB);
if (PredNode) {
assert(BBNode && "PredNode unreachable but BBNode reachable?");
for (DomTreeNode *C : to_vector(BBNode->children()))
C->setIDom(PredNode);
}
}
// DTU update: Collect all the edges that exit BB.
// These dominator edges will be redirected from Pred.
std::vector<DominatorTree::UpdateType> Updates;
if (DTU) {
assert(!DT && "cannot use both DT and DTU for updates");
// To avoid processing the same predecessor more than once.
SmallPtrSet<BasicBlock *, 8> SeenSuccs;
SmallPtrSet<BasicBlock *, 2> SuccsOfPredBB(succ_begin(PredBB),
succ_end(PredBB));
Updates.reserve(Updates.size() + 2 * succ_size(BB) + 1);
// Add insert edges first. Experimentally, for the particular case of two
// blocks that can be merged, with a single successor and single predecessor
// respectively, it is beneficial to have all insert updates first. Deleting
// edges first may lead to unreachable blocks, followed by inserting edges
// making the blocks reachable again. Such DT updates lead to high compile
// times. We add inserts before deletes here to reduce compile time.
for (BasicBlock *SuccOfBB : successors(BB))
// This successor of BB may already be a PredBB's successor.
if (!SuccsOfPredBB.contains(SuccOfBB))
if (SeenSuccs.insert(SuccOfBB).second)
Updates.push_back({DominatorTree::Insert, PredBB, SuccOfBB});
SeenSuccs.clear();
for (BasicBlock *SuccOfBB : successors(BB))
if (SeenSuccs.insert(SuccOfBB).second)
Updates.push_back({DominatorTree::Delete, BB, SuccOfBB});
Updates.push_back({DominatorTree::Delete, PredBB, BB});
}
Instruction *STI = BB->getTerminator();
Instruction *Start = &*BB->begin();
// If there's nothing to move, mark the starting instruction as the last
// instruction in the block. Terminator instruction is handled separately.
if (Start == STI)
Start = PTI;
// Move all definitions in the successor to the predecessor...
PredBB->splice(PTI->getIterator(), BB, BB->begin(), STI->getIterator());
if (MSSAU)
MSSAU->moveAllAfterMergeBlocks(BB, PredBB, Start);
// Make all PHI nodes that referred to BB now refer to Pred as their
// source...
BB->replaceAllUsesWith(PredBB);
if (PredecessorWithTwoSuccessors) {
// Delete the unconditional branch from BB.
BB->back().eraseFromParent();
// Update branch in the predecessor.
PredBB_BI->setSuccessor(FallThruPath, NewSucc);
} else {
// Delete the unconditional branch from the predecessor.
PredBB->back().eraseFromParent();
// Move terminator instruction.
BB->back().moveBeforePreserving(*PredBB, PredBB->end());
// Terminator may be a memory accessing instruction too.
if (MSSAU)
if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
MSSAU->getMemorySSA()->getMemoryAccess(PredBB->getTerminator())))
MSSAU->moveToPlace(MUD, PredBB, MemorySSA::End);
}
// Add unreachable to now empty BB.
new UnreachableInst(BB->getContext(), BB);
// Inherit predecessors name if it exists.
if (!PredBB->hasName())
PredBB->takeName(BB);
if (LI)
LI->removeBlock(BB);
if (MemDep)
MemDep->invalidateCachedPredecessors();
if (DTU)
DTU->applyUpdates(Updates);
if (DT) {
assert(succ_empty(BB) &&
"successors should have been transferred to PredBB");
DT->eraseNode(BB);
}
// Finally, erase the old block and update dominator info.
DeleteDeadBlock(BB, DTU);
return true;
}
bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L, DomTreeUpdater *DTU,
LoopInfo *LI) {
assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
bool BlocksHaveBeenMerged = false;
while (!MergeBlocks.empty()) {
BasicBlock *BB = *MergeBlocks.begin();
BasicBlock *Dest = BB->getSingleSuccessor();
if (Dest && (!L || L->contains(Dest))) {
BasicBlock *Fold = Dest->getUniquePredecessor();
(void)Fold;
if (MergeBlockIntoPredecessor(Dest, DTU, LI)) {
assert(Fold == BB &&
"Expecting BB to be unique predecessor of the Dest block");
MergeBlocks.erase(Dest);
BlocksHaveBeenMerged = true;
} else
MergeBlocks.erase(BB);
} else
MergeBlocks.erase(BB);
}
return BlocksHaveBeenMerged;
}
/// Remove redundant instructions within sequences of consecutive dbg.value
/// instructions. This is done using a backward scan to keep the last dbg.value
/// describing a specific variable/fragment.
///
/// BackwardScan strategy:
/// ----------------------
/// Given a sequence of consecutive DbgValueInst like this
///
/// dbg.value ..., "x", FragmentX1 (*)
/// dbg.value ..., "y", FragmentY1
/// dbg.value ..., "x", FragmentX2
/// dbg.value ..., "x", FragmentX1 (**)
///
/// then the instruction marked with (*) can be removed (it is guaranteed to be
/// obsoleted by the instruction marked with (**) as the latter instruction is
/// describing the same variable using the same fragment info).
///
/// Possible improvements:
/// - Check fully overlapping fragments and not only identical fragments.
/// - Support dbg.declare. dbg.label, and possibly other meta instructions being
/// part of the sequence of consecutive instructions.
static bool DPValuesRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
SmallVector<DPValue *, 8> ToBeRemoved;
SmallDenseSet<DebugVariable> VariableSet;
for (auto &I : reverse(*BB)) {
for (DbgRecord &DR : reverse(I.getDbgValueRange())) {
if (isa<DPLabel>(DR)) {
// Emulate existing behaviour (see comment below for dbg.declares).
// FIXME: Don't do this.
VariableSet.clear();
continue;
}
DPValue &DPV = cast<DPValue>(DR);
// Skip declare-type records, as the debug intrinsic method only works
// on dbg.value intrinsics.
if (DPV.getType() == DPValue::LocationType::Declare) {
// The debug intrinsic method treats dbg.declares are "non-debug"
// instructions (i.e., a break in a consecutive range of debug
// intrinsics). Emulate that to create identical outputs. See
// "Possible improvements" above.
// FIXME: Delete the line below.
VariableSet.clear();
continue;
}
DebugVariable Key(DPV.getVariable(), DPV.getExpression(),
DPV.getDebugLoc()->getInlinedAt());
auto R = VariableSet.insert(Key);
// If the same variable fragment is described more than once it is enough
// to keep the last one (i.e. the first found since we for reverse
// iteration).
if (R.second)
continue;
if (DPV.isDbgAssign()) {
// Don't delete dbg.assign intrinsics that are linked to instructions.
if (!at::getAssignmentInsts(&DPV).empty())
continue;
// Unlinked dbg.assign intrinsics can be treated like dbg.values.
}
ToBeRemoved.push_back(&DPV);
continue;
}
// Sequence with consecutive dbg.value instrs ended. Clear the map to
// restart identifying redundant instructions if case we find another
// dbg.value sequence.
VariableSet.clear();
}
for (auto &DPV : ToBeRemoved)
DPV->eraseFromParent();
return !ToBeRemoved.empty();
}
static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
if (BB->IsNewDbgInfoFormat)
return DPValuesRemoveRedundantDbgInstrsUsingBackwardScan(BB);
SmallVector<DbgValueInst *, 8> ToBeRemoved;
SmallDenseSet<DebugVariable> VariableSet;
for (auto &I : reverse(*BB)) {
if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
DebugVariable Key(DVI->getVariable(),
DVI->getExpression(),
DVI->getDebugLoc()->getInlinedAt());
auto R = VariableSet.insert(Key);
// If the variable fragment hasn't been seen before then we don't want
// to remove this dbg intrinsic.
if (R.second)
continue;
if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI)) {
// Don't delete dbg.assign intrinsics that are linked to instructions.
if (!at::getAssignmentInsts(DAI).empty())
continue;
// Unlinked dbg.assign intrinsics can be treated like dbg.values.
}
// If the same variable fragment is described more than once it is enough
// to keep the last one (i.e. the first found since we for reverse
// iteration).
ToBeRemoved.push_back(DVI);
continue;
}
// Sequence with consecutive dbg.value instrs ended. Clear the map to
// restart identifying redundant instructions if case we find another
// dbg.value sequence.
VariableSet.clear();
}
for (auto &Instr : ToBeRemoved)
Instr->eraseFromParent();
return !ToBeRemoved.empty();
}
/// Remove redundant dbg.value instructions using a forward scan. This can
/// remove a dbg.value instruction that is redundant due to indicating that a
/// variable has the same value as already being indicated by an earlier
/// dbg.value.
///
/// ForwardScan strategy:
/// ---------------------
/// Given two identical dbg.value instructions, separated by a block of
/// instructions that isn't describing the same variable, like this
///
/// dbg.value X1, "x", FragmentX1 (**)
/// <block of instructions, none being "dbg.value ..., "x", ...">
/// dbg.value X1, "x", FragmentX1 (*)
///
/// then the instruction marked with (*) can be removed. Variable "x" is already
/// described as being mapped to the SSA value X1.
///
/// Possible improvements:
/// - Keep track of non-overlapping fragments.
static bool DPValuesRemoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
SmallVector<DPValue *, 8> ToBeRemoved;
DenseMap<DebugVariable, std::pair<SmallVector<Value *, 4>, DIExpression *>>
VariableMap;
for (auto &I : *BB) {
for (DPValue &DPV : DPValue::filter(I.getDbgValueRange())) {
if (DPV.getType() == DPValue::LocationType::Declare)
continue;
DebugVariable Key(DPV.getVariable(), std::nullopt,
DPV.getDebugLoc()->getInlinedAt());
auto VMI = VariableMap.find(Key);
// A dbg.assign with no linked instructions can be treated like a
// dbg.value (i.e. can be deleted).
bool IsDbgValueKind =
(!DPV.isDbgAssign() || at::getAssignmentInsts(&DPV).empty());
// Update the map if we found a new value/expression describing the
// variable, or if the variable wasn't mapped already.
SmallVector<Value *, 4> Values(DPV.location_ops());
if (VMI == VariableMap.end() || VMI->second.first != Values ||
VMI->second.second != DPV.getExpression()) {
if (IsDbgValueKind)
VariableMap[Key] = {Values, DPV.getExpression()};
else
VariableMap[Key] = {Values, nullptr};
continue;
}
// Don't delete dbg.assign intrinsics that are linked to instructions.
if (!IsDbgValueKind)
continue;
// Found an identical mapping. Remember the instruction for later removal.
ToBeRemoved.push_back(&DPV);
}
}
for (auto *DPV : ToBeRemoved)
DPV->eraseFromParent();
return !ToBeRemoved.empty();
}
static bool DPValuesRemoveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
assert(BB->isEntryBlock() && "expected entry block");
SmallVector<DPValue *, 8> ToBeRemoved;
DenseSet<DebugVariable> SeenDefForAggregate;
// Returns the DebugVariable for DVI with no fragment info.
auto GetAggregateVariable = [](const DPValue &DPV) {
return DebugVariable(DPV.getVariable(), std::nullopt,
DPV.getDebugLoc().getInlinedAt());
};
// Remove undef dbg.assign intrinsics that are encountered before
// any non-undef intrinsics from the entry block.
for (auto &I : *BB) {
for (DPValue &DPV : DPValue::filter(I.getDbgValueRange())) {
if (!DPV.isDbgValue() && !DPV.isDbgAssign())
continue;
bool IsDbgValueKind =
(DPV.isDbgValue() || at::getAssignmentInsts(&DPV).empty());
DebugVariable Aggregate = GetAggregateVariable(DPV);
if (!SeenDefForAggregate.contains(Aggregate)) {
bool IsKill = DPV.isKillLocation() && IsDbgValueKind;
if (!IsKill) {
SeenDefForAggregate.insert(Aggregate);
} else if (DPV.isDbgAssign()) {
ToBeRemoved.push_back(&DPV);
}
}
}
}
for (DPValue *DPV : ToBeRemoved)
DPV->eraseFromParent();
return !ToBeRemoved.empty();
}
static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
if (BB->IsNewDbgInfoFormat)
return DPValuesRemoveRedundantDbgInstrsUsingForwardScan(BB);
SmallVector<DbgValueInst *, 8> ToBeRemoved;
DenseMap<DebugVariable, std::pair<SmallVector<Value *, 4>, DIExpression *>>
VariableMap;
for (auto &I : *BB) {
if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
DebugVariable Key(DVI->getVariable(), std::nullopt,
DVI->getDebugLoc()->getInlinedAt());
auto VMI = VariableMap.find(Key);
auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
// A dbg.assign with no linked instructions can be treated like a
// dbg.value (i.e. can be deleted).
bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
// Update the map if we found a new value/expression describing the
// variable, or if the variable wasn't mapped already.
SmallVector<Value *, 4> Values(DVI->getValues());
if (VMI == VariableMap.end() || VMI->second.first != Values ||
VMI->second.second != DVI->getExpression()) {
// Use a sentinel value (nullptr) for the DIExpression when we see a
// linked dbg.assign so that the next debug intrinsic will never match
// it (i.e. always treat linked dbg.assigns as if they're unique).
if (IsDbgValueKind)
VariableMap[Key] = {Values, DVI->getExpression()};
else
VariableMap[Key] = {Values, nullptr};
continue;
}
// Don't delete dbg.assign intrinsics that are linked to instructions.
if (!IsDbgValueKind)
continue;
ToBeRemoved.push_back(DVI);
}
}
for (auto &Instr : ToBeRemoved)
Instr->eraseFromParent();
return !ToBeRemoved.empty();
}
/// Remove redundant undef dbg.assign intrinsic from an entry block using a
/// forward scan.
/// Strategy:
/// ---------------------
/// Scanning forward, delete dbg.assign intrinsics iff they are undef, not
/// linked to an intrinsic, and don't share an aggregate variable with a debug
/// intrinsic that didn't meet the criteria. In other words, undef dbg.assigns
/// that come before non-undef debug intrinsics for the variable are
/// deleted. Given:
///
/// dbg.assign undef, "x", FragmentX1 (*)
/// <block of instructions, none being "dbg.value ..., "x", ...">
/// dbg.value %V, "x", FragmentX2
/// <block of instructions, none being "dbg.value ..., "x", ...">
/// dbg.assign undef, "x", FragmentX1
///
/// then (only) the instruction marked with (*) can be removed.
/// Possible improvements:
/// - Keep track of non-overlapping fragments.
static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
if (BB->IsNewDbgInfoFormat)
return DPValuesRemoveUndefDbgAssignsFromEntryBlock(BB);
assert(BB->isEntryBlock() && "expected entry block");
SmallVector<DbgAssignIntrinsic *, 8> ToBeRemoved;
DenseSet<DebugVariable> SeenDefForAggregate;
// Returns the DebugVariable for DVI with no fragment info.
auto GetAggregateVariable = [](DbgValueInst *DVI) {
return DebugVariable(DVI->getVariable(), std::nullopt,
DVI->getDebugLoc()->getInlinedAt());
};
// Remove undef dbg.assign intrinsics that are encountered before
// any non-undef intrinsics from the entry block.
for (auto &I : *BB) {
DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I);
if (!DVI)
continue;
auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
DebugVariable Aggregate = GetAggregateVariable(DVI);
if (!SeenDefForAggregate.contains(Aggregate)) {
bool IsKill = DVI->isKillLocation() && IsDbgValueKind;
if (!IsKill) {
SeenDefForAggregate.insert(Aggregate);
} else if (DAI) {
ToBeRemoved.push_back(DAI);
}
}
}
for (DbgAssignIntrinsic *DAI : ToBeRemoved)
DAI->eraseFromParent();
return !ToBeRemoved.empty();
}
bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
bool MadeChanges = false;
// By using the "backward scan" strategy before the "forward scan" strategy we
// can remove both dbg.value (2) and (3) in a situation like this:
//
// (1) dbg.value V1, "x", DIExpression()
// ...
// (2) dbg.value V2, "x", DIExpression()
// (3) dbg.value V1, "x", DIExpression()
//
// The backward scan will remove (2), it is made obsolete by (3). After
// getting (2) out of the way, the foward scan will remove (3) since "x"
// already is described as having the value V1 at (1).
MadeChanges |= removeRedundantDbgInstrsUsingBackwardScan(BB);
if (BB->isEntryBlock() &&
isAssignmentTrackingEnabled(*BB->getParent()->getParent()))
MadeChanges |= removeUndefDbgAssignsFromEntryBlock(BB);
MadeChanges |= removeRedundantDbgInstrsUsingForwardScan(BB);
if (MadeChanges)
LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
<< BB->getName() << "\n");
return MadeChanges;
}
void llvm::ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V) {
Instruction &I = *BI;
// Replaces all of the uses of the instruction with uses of the value
I.replaceAllUsesWith(V);
// Make sure to propagate a name if there is one already.
if (I.hasName() && !V->hasName())
V->takeName(&I);
// Delete the unnecessary instruction now...
BI = BI->eraseFromParent();
}
void llvm::ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
Instruction *I) {
assert(I->getParent() == nullptr &&
"ReplaceInstWithInst: Instruction already inserted into basic block!");
// Copy debug location to newly added instruction, if it wasn't already set
// by the caller.
if (!I->getDebugLoc())
I->setDebugLoc(BI->getDebugLoc());
// Insert the new instruction into the basic block...
BasicBlock::iterator New = I->insertInto(BB, BI);
// Replace all uses of the old instruction, and delete it.
ReplaceInstWithValue(BI, I);
// Move BI back to point to the newly inserted instruction
BI = New;
}
bool llvm::IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB) {
// Remember visited blocks to avoid infinite loop
SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
unsigned Depth = 0;
while (BB && Depth++ < MaxDeoptOrUnreachableSuccessorCheckDepth &&
VisitedBlocks.insert(BB).second) {
if (isa<UnreachableInst>(BB->getTerminator()) ||
BB->getTerminatingDeoptimizeCall())
return true;
BB = BB->getUniqueSuccessor();
}
return false;
}
void llvm::ReplaceInstWithInst(Instruction *From, Instruction *To) {
BasicBlock::iterator BI(From);
ReplaceInstWithInst(From->getParent(), BI, To);
}
BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, DominatorTree *DT,
LoopInfo *LI, MemorySSAUpdater *MSSAU,
const Twine &BBName) {
unsigned SuccNum = GetSuccessorNumber(BB, Succ);
Instruction *LatchTerm = BB->getTerminator();
CriticalEdgeSplittingOptions Options =
CriticalEdgeSplittingOptions(DT, LI, MSSAU).setPreserveLCSSA();
if ((isCriticalEdge(LatchTerm, SuccNum, Options.MergeIdenticalEdges))) {
// If it is a critical edge, and the succesor is an exception block, handle
// the split edge logic in this specific function
if (Succ->isEHPad())
return ehAwareSplitEdge(BB, Succ, nullptr, nullptr, Options, BBName);
// If this is a critical edge, let SplitKnownCriticalEdge do it.
return SplitKnownCriticalEdge(LatchTerm, SuccNum, Options, BBName);
}
// If the edge isn't critical, then BB has a single successor or Succ has a
// single pred. Split the block.
if (BasicBlock *SP = Succ->getSinglePredecessor()) {
// If the successor only has a single pred, split the top of the successor
// block.
assert(SP == BB && "CFG broken");
SP = nullptr;
return SplitBlock(Succ, &Succ->front(), DT, LI, MSSAU, BBName,
/*Before=*/true);
}
// Otherwise, if BB has a single successor, split it at the bottom of the
// block.
assert(BB->getTerminator()->getNumSuccessors() == 1 &&
"Should have a single succ!");
return SplitBlock(BB, BB->getTerminator(), DT, LI, MSSAU, BBName);
}
void llvm::setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ) {
if (auto *II = dyn_cast<InvokeInst>(TI))
II->setUnwindDest(Succ);
else if (auto *CS = dyn_cast<CatchSwitchInst>(TI))
CS->setUnwindDest(Succ);
else if (auto *CR = dyn_cast<CleanupReturnInst>(TI))
CR->setUnwindDest(Succ);
else
llvm_unreachable("unexpected terminator instruction");
}
void llvm::updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
BasicBlock *NewPred, PHINode *Until) {
int BBIdx = 0;
for (PHINode &PN : DestBB->phis()) {
// We manually update the LandingPadReplacement PHINode and it is the last
// PHI Node. So, if we find it, we are done.
if (Until == &PN)
break;
// Reuse the previous value of BBIdx if it lines up. In cases where we
// have multiple phi nodes with *lots* of predecessors, this is a speed
// win because we don't have to scan the PHI looking for TIBB. This
// happens because the BB list of PHI nodes are usually in the same
// order.
if (PN.getIncomingBlock(BBIdx) != OldPred)
BBIdx = PN.getBasicBlockIndex(OldPred);
assert(BBIdx != -1 && "Invalid PHI Index!");
PN.setIncomingBlock(BBIdx, NewPred);
}
}
BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
LandingPadInst *OriginalPad,
PHINode *LandingPadReplacement,
const CriticalEdgeSplittingOptions &Options,
const Twine &BBName) {
auto *PadInst = Succ->getFirstNonPHI();
if (!LandingPadReplacement && !PadInst->isEHPad())
return SplitEdge(BB, Succ, Options.DT, Options.LI, Options.MSSAU, BBName);
auto *LI = Options.LI;
SmallVector<BasicBlock *, 4> LoopPreds;
// Check if extra modifications will be required to preserve loop-simplify
// form after splitting. If it would require splitting blocks with IndirectBr
// terminators, bail out if preserving loop-simplify form is requested.
if (Options.PreserveLoopSimplify && LI) {
if (Loop *BBLoop = LI->getLoopFor(BB)) {
// The only way that we can break LoopSimplify form by splitting a
// critical edge is when there exists some edge from BBLoop to Succ *and*
// the only edge into Succ from outside of BBLoop is that of NewBB after
// the split. If the first isn't true, then LoopSimplify still holds,
// NewBB is the new exit block and it has no non-loop predecessors. If the
// second isn't true, then Succ was not in LoopSimplify form prior to
// the split as it had a non-loop predecessor. In both of these cases,
// the predecessor must be directly in BBLoop, not in a subloop, or again
// LoopSimplify doesn't hold.
for (BasicBlock *P : predecessors(Succ)) {
if (P == BB)
continue; // The new block is known.
if (LI->getLoopFor(P) != BBLoop) {
// Loop is not in LoopSimplify form, no need to re simplify after
// splitting edge.
LoopPreds.clear();
break;
}
LoopPreds.push_back(P);
}
// Loop-simplify form can be preserved, if we can split all in-loop
// predecessors.
if (any_of(LoopPreds, [](BasicBlock *Pred) {
return isa<IndirectBrInst>(Pred->getTerminator());
})) {
return nullptr;
}
}
}
auto *NewBB =
BasicBlock::Create(BB->getContext(), BBName, BB->getParent(), Succ);
setUnwindEdgeTo(BB->getTerminator(), NewBB);
updatePhiNodes(Succ, BB, NewBB, LandingPadReplacement);
if (LandingPadReplacement) {
auto *NewLP = OriginalPad->clone();
auto *Terminator = BranchInst::Create(Succ, NewBB);
NewLP->insertBefore(Terminator);
LandingPadReplacement->addIncoming(NewLP, NewBB);
} else {
Value *ParentPad = nullptr;
if (auto *FuncletPad = dyn_cast<FuncletPadInst>(PadInst))
ParentPad = FuncletPad->getParentPad();
else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(PadInst))
ParentPad = CatchSwitch->getParentPad();
else if (auto *CleanupPad = dyn_cast<CleanupPadInst>(PadInst))
ParentPad = CleanupPad->getParentPad();
else if (auto *LandingPad = dyn_cast<LandingPadInst>(PadInst))
ParentPad = LandingPad->getParent();
else
llvm_unreachable("handling for other EHPads not implemented yet");
auto *NewCleanupPad = CleanupPadInst::Create(ParentPad, {}, BBName, NewBB);
CleanupReturnInst::Create(NewCleanupPad, Succ, NewBB);
}
auto *DT = Options.DT;
auto *MSSAU = Options.MSSAU;
if (!DT && !LI)
return NewBB;
if (DT) {
DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
SmallVector<DominatorTree::UpdateType, 3> Updates;
Updates.push_back({DominatorTree::Insert, BB, NewBB});
Updates.push_back({DominatorTree::Insert, NewBB, Succ});
Updates.push_back({DominatorTree::Delete, BB, Succ});
DTU.applyUpdates(Updates);
DTU.flush();
if (MSSAU) {
MSSAU->applyUpdates(Updates, *DT);
if (VerifyMemorySSA)
MSSAU->getMemorySSA()->verifyMemorySSA();
}
}
if (LI) {
if (Loop *BBLoop = LI->getLoopFor(BB)) {
// If one or the other blocks were not in a loop, the new block is not
// either, and thus LI doesn't need to be updated.
if (Loop *SuccLoop = LI->getLoopFor(Succ)) {
if (BBLoop == SuccLoop) {
// Both in the same loop, the NewBB joins loop.
SuccLoop->addBasicBlockToLoop(NewBB, *LI);
} else if (BBLoop->contains(SuccLoop)) {
// Edge from an outer loop to an inner loop. Add to the outer loop.
BBLoop->addBasicBlockToLoop(NewBB, *LI);
} else if (SuccLoop->contains(BBLoop)) {
// Edge from an inner loop to an outer loop. Add to the outer loop.
SuccLoop->addBasicBlockToLoop(NewBB, *LI);
} else {
// Edge from two loops with no containment relation. Because these
// are natural loops, we know that the destination block must be the
// header of its loop (adding a branch into a loop elsewhere would
// create an irreducible loop).
assert(SuccLoop->getHeader() == Succ &&
"Should not create irreducible loops!");
if (Loop *P = SuccLoop->getParentLoop())
P->addBasicBlockToLoop(NewBB, *LI);
}
}
// If BB is in a loop and Succ is outside of that loop, we may need to
// update LoopSimplify form and LCSSA form.
if (!BBLoop->contains(Succ)) {
assert(!BBLoop->contains(NewBB) &&
"Split point for loop exit is contained in loop!");
// Update LCSSA form in the newly created exit block.
if (Options.PreserveLCSSA) {
createPHIsForSplitLoopExit(BB, NewBB, Succ);
}
if (!LoopPreds.empty()) {
BasicBlock *NewExitBB = SplitBlockPredecessors(
Succ, LoopPreds, "split", DT, LI, MSSAU, Options.PreserveLCSSA);
if (Options.PreserveLCSSA)
createPHIsForSplitLoopExit(LoopPreds, NewExitBB, Succ);
}
}
}
}
return NewBB;
}
void llvm::createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
BasicBlock *SplitBB, BasicBlock *DestBB) {
// SplitBB shouldn't have anything non-trivial in it yet.
assert((SplitBB->getFirstNonPHI() == SplitBB->getTerminator() ||
SplitBB->isLandingPad()) &&
"SplitBB has non-PHI nodes!");
// For each PHI in the destination block.
for (PHINode &PN : DestBB->phis()) {
int Idx = PN.getBasicBlockIndex(SplitBB);
assert(Idx >= 0 && "Invalid Block Index");
Value *V = PN.getIncomingValue(Idx);
// If the input is a PHI which already satisfies LCSSA, don't create
// a new one.
if (const PHINode *VP = dyn_cast<PHINode>(V))
if (VP->getParent() == SplitBB)
continue;
// Otherwise a new PHI is needed. Create one and populate it.
PHINode *NewPN = PHINode::Create(PN.getType(), Preds.size(), "split");
BasicBlock::iterator InsertPos =
SplitBB->isLandingPad() ? SplitBB->begin()
: SplitBB->getTerminator()->getIterator();
NewPN->insertBefore(InsertPos);
for (BasicBlock *BB : Preds)
NewPN->addIncoming(V, BB);
// Update the original PHI.
PN.setIncomingValue(Idx, NewPN);
}
}
unsigned
llvm::SplitAllCriticalEdges(Function &F,
const CriticalEdgeSplittingOptions &Options) {
unsigned NumBroken = 0;
for (BasicBlock &BB : F) {
Instruction *TI = BB.getTerminator();
if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(TI))
for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
if (SplitCriticalEdge(TI, i, Options))
++NumBroken;
}
return NumBroken;
}
static BasicBlock *SplitBlockImpl(BasicBlock *Old, BasicBlock::iterator SplitPt,
DomTreeUpdater *DTU, DominatorTree *DT,
LoopInfo *LI, MemorySSAUpdater *MSSAU,
const Twine &BBName, bool Before) {
if (Before) {
DomTreeUpdater LocalDTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
return splitBlockBefore(Old, SplitPt,
DTU ? DTU : (DT ? &LocalDTU : nullptr), LI, MSSAU,
BBName);
}
BasicBlock::iterator SplitIt = SplitPt;
while (isa<PHINode>(SplitIt) || SplitIt->isEHPad()) {
++SplitIt;
assert(SplitIt != SplitPt->getParent()->end());
}
std::string Name = BBName.str();
BasicBlock *New = Old->splitBasicBlock(
SplitIt, Name.empty() ? Old->getName() + ".split" : Name);
// The new block lives in whichever loop the old one did. This preserves
// LCSSA as well, because we force the split point to be after any PHI nodes.
if (LI)
if (Loop *L = LI->getLoopFor(Old))
L->addBasicBlockToLoop(New, *LI);
if (DTU) {
SmallVector<DominatorTree::UpdateType, 8> Updates;
// Old dominates New. New node dominates all other nodes dominated by Old.
SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld;
Updates.push_back({DominatorTree::Insert, Old, New});
Updates.reserve(Updates.size() + 2 * succ_size(New));
for (BasicBlock *SuccessorOfOld : successors(New))
if (UniqueSuccessorsOfOld.insert(SuccessorOfOld).second) {
Updates.push_back({DominatorTree::Insert, New, SuccessorOfOld});
Updates.push_back({DominatorTree::Delete, Old, SuccessorOfOld});
}
DTU->applyUpdates(Updates);
} else if (DT)
// Old dominates New. New node dominates all other nodes dominated by Old.
if (DomTreeNode *OldNode = DT->getNode(Old)) {
std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
DomTreeNode *NewNode = DT->addNewBlock(New, Old);
for (DomTreeNode *I : Children)
DT->changeImmediateDominator(I, NewNode);
}
// Move MemoryAccesses still tracked in Old, but part of New now.
// Update accesses in successor blocks accordingly.
if (MSSAU)
MSSAU->moveAllAfterSpliceBlocks(Old, New, &*(New->begin()));
return New;
}
BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
DominatorTree *DT, LoopInfo *LI,
MemorySSAUpdater *MSSAU, const Twine &BBName,
bool Before) {
return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName,
Before);
}
BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
DomTreeUpdater *DTU, LoopInfo *LI,
MemorySSAUpdater *MSSAU, const Twine &BBName,
bool Before) {
return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName,
Before);
}
BasicBlock *llvm::splitBlockBefore(BasicBlock *Old, BasicBlock::iterator SplitPt,
DomTreeUpdater *DTU, LoopInfo *LI,
MemorySSAUpdater *MSSAU,
const Twine &BBName) {
BasicBlock::iterator SplitIt = SplitPt;
while (isa<PHINode>(SplitIt) || SplitIt->isEHPad())
++SplitIt;
std::string Name = BBName.str();
BasicBlock *New = Old->splitBasicBlock(
SplitIt, Name.empty() ? Old->getName() + ".split" : Name,
/* Before=*/true);
// The new block lives in whichever loop the old one did. This preserves
// LCSSA as well, because we force the split point to be after any PHI nodes.
if (LI)
if (Loop *L = LI->getLoopFor(Old))
L->addBasicBlockToLoop(New, *LI);
if (DTU) {
SmallVector<DominatorTree::UpdateType, 8> DTUpdates;
// New dominates Old. The predecessor nodes of the Old node dominate
// New node.
SmallPtrSet<BasicBlock *, 8> UniquePredecessorsOfOld;
DTUpdates.push_back({DominatorTree::Insert, New, Old});
DTUpdates.reserve(DTUpdates.size() + 2 * pred_size(New));
for (BasicBlock *PredecessorOfOld : predecessors(New))
if (UniquePredecessorsOfOld.insert(PredecessorOfOld).second) {
DTUpdates.push_back({DominatorTree::Insert, PredecessorOfOld, New});
DTUpdates.push_back({DominatorTree::Delete, PredecessorOfOld, Old});
}
DTU->applyUpdates(DTUpdates);
// Move MemoryAccesses still tracked in Old, but part of New now.
// Update accesses in successor blocks accordingly.
if (MSSAU) {
MSSAU->applyUpdates(DTUpdates, DTU->getDomTree());
if (VerifyMemorySSA)
MSSAU->getMemorySSA()->verifyMemorySSA();
}
}
return New;
}
/// Update DominatorTree, LoopInfo, and LCCSA analysis information.
static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
ArrayRef<BasicBlock *> Preds,
DomTreeUpdater *DTU, DominatorTree *DT,
LoopInfo *LI, MemorySSAUpdater *MSSAU,
bool PreserveLCSSA, bool &HasLoopExit) {
// Update dominator tree if available.
if (DTU) {
// Recalculation of DomTree is needed when updating a forward DomTree and
// the Entry BB is replaced.
if (NewBB->isEntryBlock() && DTU->hasDomTree()) {
// The entry block was removed and there is no external interface for
// the dominator tree to be notified of this change. In this corner-case
// we recalculate the entire tree.
DTU->recalculate(*NewBB->getParent());
} else {
// Split block expects NewBB to have a non-empty set of predecessors.
SmallVector<DominatorTree::UpdateType, 8> Updates;
SmallPtrSet<BasicBlock *, 8> UniquePreds;
Updates.push_back({DominatorTree::Insert, NewBB, OldBB});
Updates.reserve(Updates.size() + 2 * Preds.size());
for (auto *Pred : Preds)
if (UniquePreds.insert(Pred).second) {
Updates.push_back({DominatorTree::Insert, Pred, NewBB});
Updates.push_back({DominatorTree::Delete, Pred, OldBB});
}
DTU->applyUpdates(Updates);
}
} else if (DT) {
if (OldBB == DT->getRootNode()->getBlock()) {
assert(NewBB->isEntryBlock());
DT->setNewRoot(NewBB);
} else {
// Split block expects NewBB to have a non-empty set of predecessors.
DT->splitBlock(NewBB);
}
}
// Update MemoryPhis after split if MemorySSA is available
if (MSSAU)
MSSAU->wireOldPredecessorsToNewImmediatePredecessor(OldBB, NewBB, Preds);
// The rest of the logic is only relevant for updating the loop structures.
if (!LI)
return;
if (DTU && DTU->hasDomTree())
DT = &DTU->getDomTree();
assert(DT && "DT should be available to update LoopInfo!");
Loop *L = LI->getLoopFor(OldBB);
// If we need to preserve loop analyses, collect some information about how
// this split will affect loops.
bool IsLoopEntry = !!L;
bool SplitMakesNewLoopHeader = false;
for (BasicBlock *Pred : Preds) {
// Preds that are not reachable from entry should not be used to identify if
// OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
// are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
// as true and make the NewBB the header of some loop. This breaks LI.
if (!DT->isReachableFromEntry(Pred))
continue;
// If we need to preserve LCSSA, determine if any of the preds is a loop
// exit.
if (PreserveLCSSA)
if (Loop *PL = LI->getLoopFor(Pred))
if (!PL->contains(OldBB))
HasLoopExit = true;
// If we need to preserve LoopInfo, note whether any of the preds crosses
// an interesting loop boundary.
if (!L)
continue;
if (L->contains(Pred))
IsLoopEntry = false;
else
SplitMakesNewLoopHeader = true;
}
// Unless we have a loop for OldBB, nothing else to do here.
if (!L)
return;
if (IsLoopEntry) {
// Add the new block to the nearest enclosing loop (and not an adjacent
// loop). To find this, examine each of the predecessors and determine which
// loops enclose them, and select the most-nested loop which contains the
// loop containing the block being split.
Loop *InnermostPredLoop = nullptr;
for (BasicBlock *Pred : Preds) {
if (Loop *PredLoop = LI->getLoopFor(Pred)) {
// Seek a loop which actually contains the block being split (to avoid
// adjacent loops).
while (PredLoop && !PredLoop->contains(OldBB))
PredLoop = PredLoop->getParentLoop();
// Select the most-nested of these loops which contains the block.
if (PredLoop && PredLoop->contains(OldBB) &&
(!InnermostPredLoop ||
InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
InnermostPredLoop = PredLoop;
}
}
if (InnermostPredLoop)
InnermostPredLoop->addBasicBlockToLoop(NewBB, *LI);
} else {
L->addBasicBlockToLoop(NewBB, *LI);
if (SplitMakesNewLoopHeader)
L->moveToHeader(NewBB);
}
}
/// Update the PHI nodes in OrigBB to include the values coming from NewBB.
/// This also updates AliasAnalysis, if available.
static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
ArrayRef<BasicBlock *> Preds, BranchInst *BI,
bool HasLoopExit) {
// Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
SmallPtrSet<BasicBlock *, 16> PredSet(Preds.begin(), Preds.end());
for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(I); ) {
PHINode *PN = cast<PHINode>(I++);
// Check to see if all of the values coming in are the same. If so, we
// don't need to create a new PHI node, unless it's needed for LCSSA.
Value *InVal = nullptr;
if (!HasLoopExit) {
InVal = PN->getIncomingValueForBlock(Preds[0]);
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
if (!PredSet.count(PN->getIncomingBlock(i)))
continue;
if (!InVal)
InVal = PN->getIncomingValue(i);
else if (InVal != PN->getIncomingValue(i)) {
InVal = nullptr;
break;
}
}
}
if (InVal) {
// If all incoming values for the new PHI would be the same, just don't
// make a new PHI. Instead, just remove the incoming values from the old
// PHI.
PN->removeIncomingValueIf(
[&](unsigned Idx) {
return PredSet.contains(PN->getIncomingBlock(Idx));
},
/* DeletePHIIfEmpty */ false);
// Add an incoming value to the PHI node in the loop for the preheader
// edge.
PN->addIncoming(InVal, NewBB);
continue;
}
// If the values coming into the block are not the same, we need a new
// PHI.
// Create the new PHI node, insert it into NewBB at the end of the block
PHINode *NewPHI =
PHINode::Create(PN->getType(), Preds.size(), PN->getName() + ".ph", BI);
// NOTE! This loop walks backwards for a reason! First off, this minimizes
// the cost of removal if we end up removing a large number of values, and
// second off, this ensures that the indices for the incoming values aren't
// invalidated when we remove one.
for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
BasicBlock *IncomingBB = PN->getIncomingBlock(i);
if (PredSet.count(IncomingBB)) {
Value *V = PN->removeIncomingValue(i, false);
NewPHI->addIncoming(V, IncomingBB);
}
}
PN->addIncoming(NewPHI, NewBB);
}
}
static void SplitLandingPadPredecessorsImpl(
BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
static BasicBlock *
SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
const char *Suffix, DomTreeUpdater *DTU,
DominatorTree *DT, LoopInfo *LI,
MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
// Do not attempt to split that which cannot be split.
if (!BB->canSplitPredecessors())
return nullptr;
// For the landingpads we need to act a bit differently.
// Delegate this work to the SplitLandingPadPredecessors.
if (BB->isLandingPad()) {
SmallVector<BasicBlock*, 2> NewBBs;
std::string NewName = std::string(Suffix) + ".split-lp";
SplitLandingPadPredecessorsImpl(BB, Preds, Suffix, NewName.c_str(), NewBBs,
DTU, DT, LI, MSSAU, PreserveLCSSA);
return NewBBs[0];
}
// Create new basic block, insert right before the original block.
BasicBlock *NewBB = BasicBlock::Create(
BB->getContext(), BB->getName() + Suffix, BB->getParent(), BB);
// The new block unconditionally branches to the old block.
BranchInst *BI = BranchInst::Create(BB, NewBB);
Loop *L = nullptr;
BasicBlock *OldLatch = nullptr;
// Splitting the predecessors of a loop header creates a preheader block.
if (LI && LI->isLoopHeader(BB)) {
L = LI->getLoopFor(BB);
// Using the loop start line number prevents debuggers stepping into the
// loop body for this instruction.
BI->setDebugLoc(L->getStartLoc());
// If BB is the header of the Loop, it is possible that the loop is
// modified, such that the current latch does not remain the latch of the
// loop. If that is the case, the loop metadata from the current latch needs
// to be applied to the new latch.
OldLatch = L->getLoopLatch();
} else
BI->setDebugLoc(BB->getFirstNonPHIOrDbg()->getDebugLoc());
// Move the edges from Preds to point to NewBB instead of BB.
for (BasicBlock *Pred : Preds) {
// This is slightly more strict than necessary; the minimum requirement
// is that there be no more than one indirectbr branching to BB. And
// all BlockAddress uses would need to be updated.
assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
"Cannot split an edge from an IndirectBrInst");
Pred->getTerminator()->replaceSuccessorWith(BB, NewBB);
}
// Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
// node becomes an incoming value for BB's phi node. However, if the Preds
// list is empty, we need to insert dummy entries into the PHI nodes in BB to
// account for the newly created predecessor.
if (Preds.empty()) {
// Insert dummy values as the incoming value.
for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++I)
cast<PHINode>(I)->addIncoming(PoisonValue::get(I->getType()), NewBB);
}
// Update DominatorTree, LoopInfo, and LCCSA analysis information.
bool HasLoopExit = false;
UpdateAnalysisInformation(BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
HasLoopExit);
if (!Preds.empty()) {
// Update the PHI nodes in BB with the values coming from NewBB.
UpdatePHINodes(BB, NewBB, Preds, BI, HasLoopExit);
}
if (OldLatch) {
BasicBlock *NewLatch = L->getLoopLatch();
if (NewLatch != OldLatch) {
MDNode *MD = OldLatch->getTerminator()->getMetadata("llvm.loop");
NewLatch->getTerminator()->setMetadata("llvm.loop", MD);
// It's still possible that OldLatch is the latch of another inner loop,
// in which case we do not remove the metadata.
Loop *IL = LI->getLoopFor(OldLatch);
if (IL && IL->getLoopLatch() != OldLatch)
OldLatch->getTerminator()->setMetadata("llvm.loop", nullptr);
}
}
return NewBB;
}
BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
ArrayRef<BasicBlock *> Preds,
const char *Suffix, DominatorTree *DT,
LoopInfo *LI, MemorySSAUpdater *MSSAU,
bool PreserveLCSSA) {
return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
MSSAU, PreserveLCSSA);
}
BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
ArrayRef<BasicBlock *> Preds,
const char *Suffix,
DomTreeUpdater *DTU, LoopInfo *LI,
MemorySSAUpdater *MSSAU,
bool PreserveLCSSA) {
return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
/*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
}
static void SplitLandingPadPredecessorsImpl(
BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
// Create a new basic block for OrigBB's predecessors listed in Preds. Insert
// it right before the original block.
BasicBlock *NewBB1 = BasicBlock::Create(OrigBB->getContext(),
OrigBB->getName() + Suffix1,
OrigBB->getParent(), OrigBB);
NewBBs.push_back(NewBB1);
// The new block unconditionally branches to the old block.
BranchInst *BI1 = BranchInst::Create(OrigBB, NewBB1);
BI1->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
// Move the edges from Preds to point to NewBB1 instead of OrigBB.
for (BasicBlock *Pred : Preds) {
// This is slightly more strict than necessary; the minimum requirement
// is that there be no more than one indirectbr branching to BB. And
// all BlockAddress uses would need to be updated.
assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
"Cannot split an edge from an IndirectBrInst");
Pred->getTerminator()->replaceUsesOfWith(OrigBB, NewBB1);
}
bool HasLoopExit = false;
UpdateAnalysisInformation(OrigBB, NewBB1, Preds, DTU, DT, LI, MSSAU,
PreserveLCSSA, HasLoopExit);
// Update the PHI nodes in OrigBB with the values coming from NewBB1.
UpdatePHINodes(OrigBB, NewBB1, Preds, BI1, HasLoopExit);
// Move the remaining edges from OrigBB to point to NewBB2.
SmallVector<BasicBlock*, 8> NewBB2Preds;
for (pred_iterator i = pred_begin(OrigBB), e = pred_end(OrigBB);
i != e; ) {
BasicBlock *Pred = *i++;
if (Pred == NewBB1) continue;
assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
"Cannot split an edge from an IndirectBrInst");
NewBB2Preds.push_back(Pred);
e = pred_end(OrigBB);
}
BasicBlock *NewBB2 = nullptr;
if (!NewBB2Preds.empty()) {
// Create another basic block for the rest of OrigBB's predecessors.
NewBB2 = BasicBlock::Create(OrigBB->getContext(),
OrigBB->getName() + Suffix2,
OrigBB->getParent(), OrigBB);
NewBBs.push_back(NewBB2);
// The new block unconditionally branches to the old block.
BranchInst *BI2 = BranchInst::Create(OrigBB, NewBB2);
BI2->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
// Move the remaining edges from OrigBB to point to NewBB2.
for (BasicBlock *NewBB2Pred : NewBB2Preds)
NewBB2Pred->getTerminator()->replaceUsesOfWith(OrigBB, NewBB2);
// Update DominatorTree, LoopInfo, and LCCSA analysis information.
HasLoopExit = false;
UpdateAnalysisInformation(OrigBB, NewBB2, NewBB2Preds, DTU, DT, LI, MSSAU,
PreserveLCSSA, HasLoopExit);
// Update the PHI nodes in OrigBB with the values coming from NewBB2.
UpdatePHINodes(OrigBB, NewBB2, NewBB2Preds, BI2, HasLoopExit);
}
LandingPadInst *LPad = OrigBB->getLandingPadInst();
Instruction *Clone1 = LPad->clone();
Clone1->setName(Twine("lpad") + Suffix1);
Clone1->insertInto(NewBB1, NewBB1->getFirstInsertionPt());
if (NewBB2) {
Instruction *Clone2 = LPad->clone();
Clone2->setName(Twine("lpad") + Suffix2);
Clone2->insertInto(NewBB2, NewBB2->getFirstInsertionPt());
// Create a PHI node for the two cloned landingpad instructions only
// if the original landingpad instruction has some uses.
if (!LPad->use_empty()) {
assert(!LPad->getType()->isTokenTy() &&
"Split cannot be applied if LPad is token type. Otherwise an "
"invalid PHINode of token type would be created.");
PHINode *PN = PHINode::Create(LPad->getType(), 2, "lpad.phi", LPad);
PN->addIncoming(Clone1, NewBB1);
PN->addIncoming(Clone2, NewBB2);
LPad->replaceAllUsesWith(PN);
}
LPad->eraseFromParent();
} else {
// There is no second clone. Just replace the landing pad with the first
// clone.
LPad->replaceAllUsesWith(Clone1);
LPad->eraseFromParent();
}
}
void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
ArrayRef<BasicBlock *> Preds,
const char *Suffix1, const char *Suffix2,
SmallVectorImpl<BasicBlock *> &NewBBs,
DomTreeUpdater *DTU, LoopInfo *LI,
MemorySSAUpdater *MSSAU,
bool PreserveLCSSA) {
return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
PreserveLCSSA);
}
ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
BasicBlock *Pred,
DomTreeUpdater *DTU) {
Instruction *UncondBranch = Pred->getTerminator();
// Clone the return and add it to the end of the predecessor.
Instruction *NewRet = RI->clone();
NewRet->insertInto(Pred, Pred->end());
// If the return instruction returns a value, and if the value was a
// PHI node in "BB", propagate the right value into the return.
for (Use &Op : NewRet->operands()) {
Value *V = Op;
Instruction *NewBC = nullptr;
if (BitCastInst *BCI = dyn_cast<BitCastInst>(V)) {
// Return value might be bitcasted. Clone and insert it before the
// return instruction.
V = BCI->getOperand(0);
NewBC = BCI->clone();
NewBC->insertInto(Pred, NewRet->getIterator());
Op = NewBC;
}
Instruction *NewEV = nullptr;
if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(V)) {
V = EVI->getOperand(0);
NewEV = EVI->clone();
if (NewBC) {
NewBC->setOperand(0, NewEV);
NewEV->insertInto(Pred, NewBC->getIterator());
} else {
NewEV->insertInto(Pred, NewRet->getIterator());
Op = NewEV;
}
}
if (PHINode *PN = dyn_cast<PHINode>(V)) {
if (PN->getParent() == BB) {
if (NewEV) {
NewEV->setOperand(0, PN->getIncomingValueForBlock(Pred));
} else if (NewBC)
NewBC->setOperand(0, PN->getIncomingValueForBlock(Pred));
else
Op = PN->getIncomingValueForBlock(Pred);
}
}
}
// Update any PHI nodes in the returning block to realize that we no
// longer branch to them.
BB->removePredecessor(Pred);
UncondBranch->eraseFromParent();
if (DTU)
DTU->applyUpdates({{DominatorTree::Delete, Pred, BB}});
return cast<ReturnInst>(NewRet);
}
Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
BasicBlock::iterator SplitBefore,
bool Unreachable,
MDNode *BranchWeights,
DomTreeUpdater *DTU, LoopInfo *LI,
BasicBlock *ThenBlock) {
SplitBlockAndInsertIfThenElse(
Cond, SplitBefore, &ThenBlock, /* ElseBlock */ nullptr,
/* UnreachableThen */ Unreachable,
/* UnreachableElse */ false, BranchWeights, DTU, LI);
return ThenBlock->getTerminator();
}
Instruction *llvm::SplitBlockAndInsertIfElse(Value *Cond,
BasicBlock::iterator SplitBefore,
bool Unreachable,
MDNode *BranchWeights,
DomTreeUpdater *DTU, LoopInfo *LI,
BasicBlock *ElseBlock) {
SplitBlockAndInsertIfThenElse(
Cond, SplitBefore, /* ThenBlock */ nullptr, &ElseBlock,
/* UnreachableThen */ false,
/* UnreachableElse */ Unreachable, BranchWeights, DTU, LI);
return ElseBlock->getTerminator();
}
void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore,
Instruction **ThenTerm,
Instruction **ElseTerm,
MDNode *BranchWeights,
DomTreeUpdater *DTU, LoopInfo *LI) {
BasicBlock *ThenBlock = nullptr;
BasicBlock *ElseBlock = nullptr;
SplitBlockAndInsertIfThenElse(
Cond, SplitBefore, &ThenBlock, &ElseBlock, /* UnreachableThen */ false,
/* UnreachableElse */ false, BranchWeights, DTU, LI);
*ThenTerm = ThenBlock->getTerminator();
*ElseTerm = ElseBlock->getTerminator();
}
void llvm::SplitBlockAndInsertIfThenElse(
Value *Cond, BasicBlock::iterator SplitBefore, BasicBlock **ThenBlock,
BasicBlock **ElseBlock, bool UnreachableThen, bool UnreachableElse,
MDNode *BranchWeights, DomTreeUpdater *DTU, LoopInfo *LI) {
assert((ThenBlock || ElseBlock) &&
"At least one branch block must be created");
assert((!UnreachableThen || !UnreachableElse) &&
"Split block tail must be reachable");
SmallVector<DominatorTree::UpdateType, 8> Updates;
SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors;
BasicBlock *Head = SplitBefore->getParent();
if (DTU) {
UniqueOrigSuccessors.insert(succ_begin(Head), succ_end(Head));
Updates.reserve(4 + 2 * UniqueOrigSuccessors.size());
}
LLVMContext &C = Head->getContext();
BasicBlock *Tail = Head->splitBasicBlock(SplitBefore);
BasicBlock *TrueBlock = Tail;
BasicBlock *FalseBlock = Tail;
bool ThenToTailEdge = false;
bool ElseToTailEdge = false;
// Encapsulate the logic around creation/insertion/etc of a new block.
auto handleBlock = [&](BasicBlock **PBB, bool Unreachable, BasicBlock *&BB,
bool &ToTailEdge) {
if (PBB == nullptr)
return; // Do not create/insert a block.
if (*PBB)
BB = *PBB; // Caller supplied block, use it.
else {
// Create a new block.
BB = BasicBlock::Create(C, "", Head->getParent(), Tail);
if (Unreachable)
(void)new UnreachableInst(C, BB);
else {
(void)BranchInst::Create(Tail, BB);
ToTailEdge = true;
}
BB->getTerminator()->setDebugLoc(SplitBefore->getDebugLoc());
// Pass the new block back to the caller.
*PBB = BB;
}
};
handleBlock(ThenBlock, UnreachableThen, TrueBlock, ThenToTailEdge);
handleBlock(ElseBlock, UnreachableElse, FalseBlock, ElseToTailEdge);
Instruction *HeadOldTerm = Head->getTerminator();
BranchInst *HeadNewTerm =
BranchInst::Create(/*ifTrue*/ TrueBlock, /*ifFalse*/ FalseBlock, Cond);
HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
if (DTU) {
Updates.emplace_back(DominatorTree::Insert, Head, TrueBlock);
Updates.emplace_back(DominatorTree::Insert, Head, FalseBlock);
if (ThenToTailEdge)
Updates.emplace_back(DominatorTree::Insert, TrueBlock, Tail);
if (ElseToTailEdge)
Updates.emplace_back(DominatorTree::Insert, FalseBlock, Tail);
for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
Updates.emplace_back(DominatorTree::Insert, Tail, UniqueOrigSuccessor);
for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
Updates.emplace_back(DominatorTree::Delete, Head, UniqueOrigSuccessor);
DTU->applyUpdates(Updates);
}
if (LI) {
if (Loop *L = LI->getLoopFor(Head); L) {
if (ThenToTailEdge)
L->addBasicBlockToLoop(TrueBlock, *LI);
if (ElseToTailEdge)
L->addBasicBlockToLoop(FalseBlock, *LI);
L->addBasicBlockToLoop(Tail, *LI);
}
}
}
std::pair<Instruction*, Value*>
llvm::SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore) {
BasicBlock *LoopPred = SplitBefore->getParent();
BasicBlock *LoopBody = SplitBlock(SplitBefore->getParent(), SplitBefore);
BasicBlock *LoopExit = SplitBlock(SplitBefore->getParent(), SplitBefore);
auto *Ty = End->getType();
auto &DL = SplitBefore->getModule()->getDataLayout();
const unsigned Bitwidth = DL.getTypeSizeInBits(Ty);
IRBuilder<> Builder(LoopBody->getTerminator());
auto *IV = Builder.CreatePHI(Ty, 2, "iv");
auto *IVNext =
Builder.CreateAdd(IV, ConstantInt::get(Ty, 1), IV->getName() + ".next",
/*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
auto *IVCheck = Builder.CreateICmpEQ(IVNext, End,
IV->getName() + ".check");
Builder.CreateCondBr(IVCheck, LoopExit, LoopBody);
LoopBody->getTerminator()->eraseFromParent();
// Populate the IV PHI.
IV->addIncoming(ConstantInt::get(Ty, 0), LoopPred);
IV->addIncoming(IVNext, LoopBody);
return std::make_pair(LoopBody->getFirstNonPHI(), IV);
}
void llvm::SplitBlockAndInsertForEachLane(ElementCount EC,
Type *IndexTy, Instruction *InsertBefore,
std::function<void(IRBuilderBase&, Value*)> Func) {
IRBuilder<> IRB(InsertBefore);
if (EC.isScalable()) {
Value *NumElements = IRB.CreateElementCount(IndexTy, EC);
auto [BodyIP, Index] =
SplitBlockAndInsertSimpleForLoop(NumElements, InsertBefore);
IRB.SetInsertPoint(BodyIP);
Func(IRB, Index);
return;
}
unsigned Num = EC.getFixedValue();
for (unsigned Idx = 0; Idx < Num; ++Idx) {
IRB.SetInsertPoint(InsertBefore);
Func(IRB, ConstantInt::get(IndexTy, Idx));
}
}
void llvm::SplitBlockAndInsertForEachLane(
Value *EVL, Instruction *InsertBefore,
std::function<void(IRBuilderBase &, Value *)> Func) {
IRBuilder<> IRB(InsertBefore);
Type *Ty = EVL->getType();
if (!isa<ConstantInt>(EVL)) {
auto [BodyIP, Index] = SplitBlockAndInsertSimpleForLoop(EVL, InsertBefore);
IRB.SetInsertPoint(BodyIP);
Func(IRB, Index);
return;
}
unsigned Num = cast<ConstantInt>(EVL)->getZExtValue();
for (unsigned Idx = 0; Idx < Num; ++Idx) {
IRB.SetInsertPoint(InsertBefore);
Func(IRB, ConstantInt::get(Ty, Idx));
}
}
BranchInst *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
BasicBlock *&IfFalse) {
PHINode *SomePHI = dyn_cast<PHINode>(BB->begin());
BasicBlock *Pred1 = nullptr;
BasicBlock *Pred2 = nullptr;
if (SomePHI) {
if (SomePHI->getNumIncomingValues() != 2)
return nullptr;
Pred1 = SomePHI->getIncomingBlock(0);
Pred2 = SomePHI->getIncomingBlock(1);
} else {
pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
if (PI == PE) // No predecessor
return nullptr;
Pred1 = *PI++;
if (PI == PE) // Only one predecessor
return nullptr;
Pred2 = *PI++;
if (PI != PE) // More than two predecessors
return nullptr;
}
// We can only handle branches. Other control flow will be lowered to
// branches if possible anyway.
BranchInst *Pred1Br = dyn_cast<BranchInst>(Pred1->getTerminator());
BranchInst *Pred2Br = dyn_cast<BranchInst>(Pred2->getTerminator());
if (!Pred1Br || !Pred2Br)
return nullptr;
// Eliminate code duplication by ensuring that Pred1Br is conditional if
// either are.
if (Pred2Br->isConditional()) {
// If both branches are conditional, we don't have an "if statement". In
// reality, we could transform this case, but since the condition will be
// required anyway, we stand no chance of eliminating it, so the xform is
// probably not profitable.
if (Pred1Br->isConditional())
return nullptr;
std::swap(Pred1, Pred2);
std::swap(Pred1Br, Pred2Br);
}
if (Pred1Br->isConditional()) {
// The only thing we have to watch out for here is to make sure that Pred2
// doesn't have incoming edges from other blocks. If it does, the condition
// doesn't dominate BB.
if (!Pred2->getSinglePredecessor())
return nullptr;
// If we found a conditional branch predecessor, make sure that it branches
// to BB and Pred2Br. If it doesn't, this isn't an "if statement".
if (Pred1Br->getSuccessor(0) == BB &&
Pred1Br->getSuccessor(1) == Pred2) {
IfTrue = Pred1;
IfFalse = Pred2;
} else if (Pred1Br->getSuccessor(0) == Pred2 &&
Pred1Br->getSuccessor(1) == BB) {
IfTrue = Pred2;
IfFalse = Pred1;
} else {
// We know that one arm of the conditional goes to BB, so the other must
// go somewhere unrelated, and this must not be an "if statement".
return nullptr;
}
return Pred1Br;
}
// Ok, if we got here, both predecessors end with an unconditional branch to
// BB. Don't panic! If both blocks only have a single (identical)
// predecessor, and THAT is a conditional branch, then we're all ok!
BasicBlock *CommonPred = Pred1->getSinglePredecessor();
if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
return nullptr;
// Otherwise, if this is a conditional branch, then we can use it!
BranchInst *BI = dyn_cast<BranchInst>(CommonPred->getTerminator());
if (!BI) return nullptr;
assert(BI->isConditional() && "Two successors but not conditional?");
if (BI->getSuccessor(0) == Pred1) {
IfTrue = Pred1;
IfFalse = Pred2;
} else {
IfTrue = Pred2;
IfFalse = Pred1;
}
return BI;
}
// After creating a control flow hub, the operands of PHINodes in an outgoing
// block Out no longer match the predecessors of that block. Predecessors of Out
// that are incoming blocks to the hub are now replaced by just one edge from
// the hub. To match this new control flow, the corresponding values from each
// PHINode must now be moved a new PHINode in the first guard block of the hub.
//
// This operation cannot be performed with SSAUpdater, because it involves one
// new use: If the block Out is in the list of Incoming blocks, then the newly
// created PHI in the Hub will use itself along that edge from Out to Hub.
static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
const SetVector<BasicBlock *> &Incoming,
BasicBlock *FirstGuardBlock) {
auto I = Out->begin();
while (I != Out->end() && isa<PHINode>(I)) {
auto Phi = cast<PHINode>(I);
auto NewPhi =
PHINode::Create(Phi->getType(), Incoming.size(),
Phi->getName() + ".moved", &FirstGuardBlock->front());
for (auto *In : Incoming) {
Value *V = UndefValue::get(Phi->getType());
if (In == Out) {
V = NewPhi;
} else if (Phi->getBasicBlockIndex(In) != -1) {
V = Phi->removeIncomingValue(In, false);
}
NewPhi->addIncoming(V, In);
}
assert(NewPhi->getNumIncomingValues() == Incoming.size());
if (Phi->getNumOperands() == 0) {
Phi->replaceAllUsesWith(NewPhi);
I = Phi->eraseFromParent();
continue;
}
Phi->addIncoming(NewPhi, GuardBlock);
++I;
}
}
using BBPredicates = DenseMap<BasicBlock *, Instruction *>;
using BBSetVector = SetVector<BasicBlock *>;
// Redirects the terminator of the incoming block to the first guard
// block in the hub. The condition of the original terminator (if it
// was conditional) and its original successors are returned as a
// tuple <condition, succ0, succ1>. The function additionally filters
// out successors that are not in the set of outgoing blocks.
//
// - condition is non-null iff the branch is conditional.
// - Succ1 is non-null iff the sole/taken target is an outgoing block.
// - Succ2 is non-null iff condition is non-null and the fallthrough
// target is an outgoing block.
static std::tuple<Value *, BasicBlock *, BasicBlock *>
redirectToHub(BasicBlock *BB, BasicBlock *FirstGuardBlock,
const BBSetVector &Outgoing) {
assert(isa<BranchInst>(BB->getTerminator()) &&
"Only support branch terminator.");
auto Branch = cast<BranchInst>(BB->getTerminator());
auto Condition = Branch->isConditional() ? Branch->getCondition() : nullptr;
BasicBlock *Succ0 = Branch->getSuccessor(0);
BasicBlock *Succ1 = nullptr;
Succ0 = Outgoing.count(Succ0) ? Succ0 : nullptr;
if (Branch->isUnconditional()) {
Branch->setSuccessor(0, FirstGuardBlock);
assert(Succ0);
} else {
Succ1 = Branch->getSuccessor(1);
Succ1 = Outgoing.count(Succ1) ? Succ1 : nullptr;
assert(Succ0 || Succ1);
if (Succ0 && !Succ1) {
Branch->setSuccessor(0, FirstGuardBlock);
} else if (Succ1 && !Succ0) {
Branch->setSuccessor(1, FirstGuardBlock);
} else {
Branch->eraseFromParent();
BranchInst::Create(FirstGuardBlock, BB);
}
}
assert(Succ0 || Succ1);
return std::make_tuple(Condition, Succ0, Succ1);
}
// Setup the branch instructions for guard blocks.
//
// Each guard block terminates in a conditional branch that transfers
// control to the corresponding outgoing block or the next guard
// block. The last guard block has two outgoing blocks as successors
// since the condition for the final outgoing block is trivially
// true. So we create one less block (including the first guard block)
// than the number of outgoing blocks.
static void setupBranchForGuard(SmallVectorImpl<BasicBlock *> &GuardBlocks,
const BBSetVector &Outgoing,
BBPredicates &GuardPredicates) {
// To help keep the loop simple, temporarily append the last
// outgoing block to the list of guard blocks.
GuardBlocks.push_back(Outgoing.back());
for (int i = 0, e = GuardBlocks.size() - 1; i != e; ++i) {
auto Out = Outgoing[i];
assert(GuardPredicates.count(Out));
BranchInst::Create(Out, GuardBlocks[i + 1], GuardPredicates[Out],
GuardBlocks[i]);
}
// Remove the last block from the guard list.
GuardBlocks.pop_back();
}
/// We are using one integer to represent the block we are branching to. Then at
/// each guard block, the predicate was calcuated using a simple `icmp eq`.
static void calcPredicateUsingInteger(
const BBSetVector &Incoming, const BBSetVector &Outgoing,
SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates) {
auto &Context = Incoming.front()->getContext();
auto FirstGuardBlock = GuardBlocks.front();
auto Phi = PHINode::Create(Type::getInt32Ty(Context), Incoming.size(),
"merged.bb.idx", FirstGuardBlock);
for (auto In : Incoming) {
Value *Condition;
BasicBlock *Succ0;
BasicBlock *Succ1;
std::tie(Condition, Succ0, Succ1) =
redirectToHub(In, FirstGuardBlock, Outgoing);
Value *IncomingId = nullptr;
if (Succ0 && Succ1) {
// target_bb_index = Condition ? index_of_succ0 : index_of_succ1.
auto Succ0Iter = find(Outgoing, Succ0);
auto Succ1Iter = find(Outgoing, Succ1);
Value *Id0 = ConstantInt::get(Type::getInt32Ty(Context),
std::distance(Outgoing.begin(), Succ0Iter));
Value *Id1 = ConstantInt::get(Type::getInt32Ty(Context),
std::distance(Outgoing.begin(), Succ1Iter));
IncomingId = SelectInst::Create(Condition, Id0, Id1, "target.bb.idx",
In->getTerminator());
} else {
// Get the index of the non-null successor.
auto SuccIter = Succ0 ? find(Outgoing, Succ0) : find(Outgoing, Succ1);
IncomingId = ConstantInt::get(Type::getInt32Ty(Context),
std::distance(Outgoing.begin(), SuccIter));
}
Phi->addIncoming(IncomingId, In);
}
for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
auto Out = Outgoing[i];
auto Cmp = ICmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, Phi,
ConstantInt::get(Type::getInt32Ty(Context), i),
Out->getName() + ".predicate", GuardBlocks[i]);
GuardPredicates[Out] = Cmp;
}
}
/// We record the predicate of each outgoing block using a phi of boolean.
static void calcPredicateUsingBooleans(
const BBSetVector &Incoming, const BBSetVector &Outgoing,
SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates,
SmallVectorImpl<WeakVH> &DeletionCandidates) {
auto &Context = Incoming.front()->getContext();
auto BoolTrue = ConstantInt::getTrue(Context);
auto BoolFalse = ConstantInt::getFalse(Context);
auto FirstGuardBlock = GuardBlocks.front();
// The predicate for the last outgoing is trivially true, and so we
// process only the first N-1 successors.
for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
auto Out = Outgoing[i];
LLVM_DEBUG(dbgs() << "Creating guard for " << Out->getName() << "\n");
auto Phi =
PHINode::Create(Type::getInt1Ty(Context), Incoming.size(),
StringRef("Guard.") + Out->getName(), FirstGuardBlock);
GuardPredicates[Out] = Phi;
}
for (auto *In : Incoming) {
Value *Condition;
BasicBlock *Succ0;
BasicBlock *Succ1;
std::tie(Condition, Succ0, Succ1) =
redirectToHub(In, FirstGuardBlock, Outgoing);
// Optimization: Consider an incoming block A with both successors
// Succ0 and Succ1 in the set of outgoing blocks. The predicates
// for Succ0 and Succ1 complement each other. If Succ0 is visited
// first in the loop below, control will branch to Succ0 using the
// corresponding predicate. But if that branch is not taken, then
// control must reach Succ1, which means that the incoming value of
// the predicate from `In` is true for Succ1.
bool OneSuccessorDone = false;
for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
auto Out = Outgoing[i];
PHINode *Phi = cast<PHINode>(GuardPredicates[Out]);
if (Out != Succ0 && Out != Succ1) {
Phi->addIncoming(BoolFalse, In);
} else if (!Succ0 || !Succ1 || OneSuccessorDone) {
// Optimization: When only one successor is an outgoing block,
// the incoming predicate from `In` is always true.
Phi->addIncoming(BoolTrue, In);
} else {
assert(Succ0 && Succ1);
if (Out == Succ0) {
Phi->addIncoming(Condition, In);
} else {
auto Inverted = invertCondition(Condition);
DeletionCandidates.push_back(Condition);
Phi->addIncoming(Inverted, In);
}
OneSuccessorDone = true;
}
}
}
}
// Capture the existing control flow as guard predicates, and redirect
// control flow from \p Incoming block through the \p GuardBlocks to the
// \p Outgoing blocks.
//
// There is one guard predicate for each outgoing block OutBB. The
// predicate represents whether the hub should transfer control flow
// to OutBB. These predicates are NOT ORTHOGONAL. The Hub evaluates
// them in the same order as the Outgoing set-vector, and control
// branches to the first outgoing block whose predicate evaluates to true.
static void
convertToGuardPredicates(SmallVectorImpl<BasicBlock *> &GuardBlocks,
SmallVectorImpl<WeakVH> &DeletionCandidates,
const BBSetVector &Incoming,
const BBSetVector &Outgoing, const StringRef Prefix,
std::optional<unsigned> MaxControlFlowBooleans) {
BBPredicates GuardPredicates;
auto F = Incoming.front()->getParent();
for (int i = 0, e = Outgoing.size() - 1; i != e; ++i)
GuardBlocks.push_back(
BasicBlock::Create(F->getContext(), Prefix + ".guard", F));
// When we are using an integer to record which target block to jump to, we
// are creating less live values, actually we are using one single integer to
// store the index of the target block. When we are using booleans to store
// the branching information, we need (N-1) boolean values, where N is the
// number of outgoing block.
if (!MaxControlFlowBooleans || Outgoing.size() <= *MaxControlFlowBooleans)
calcPredicateUsingBooleans(Incoming, Outgoing, GuardBlocks, GuardPredicates,
DeletionCandidates);
else
calcPredicateUsingInteger(Incoming, Outgoing, GuardBlocks, GuardPredicates);
setupBranchForGuard(GuardBlocks, Outgoing, GuardPredicates);
}
BasicBlock *llvm::CreateControlFlowHub(
DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
const BBSetVector &Incoming, const BBSetVector &Outgoing,
const StringRef Prefix, std::optional<unsigned> MaxControlFlowBooleans) {
if (Outgoing.size() < 2)
return Outgoing.front();
SmallVector<DominatorTree::UpdateType, 16> Updates;
if (DTU) {
for (auto *In : Incoming) {
for (auto Succ : successors(In))
if (Outgoing.count(Succ))
Updates.push_back({DominatorTree::Delete, In, Succ});
}
}
SmallVector<WeakVH, 8> DeletionCandidates;
convertToGuardPredicates(GuardBlocks, DeletionCandidates, Incoming, Outgoing,
Prefix, MaxControlFlowBooleans);
auto FirstGuardBlock = GuardBlocks.front();
// Update the PHINodes in each outgoing block to match the new control flow.
for (int i = 0, e = GuardBlocks.size(); i != e; ++i)
reconnectPhis(Outgoing[i], GuardBlocks[i], Incoming, FirstGuardBlock);
reconnectPhis(Outgoing.back(), GuardBlocks.back(), Incoming, FirstGuardBlock);
if (DTU) {
int NumGuards = GuardBlocks.size();
assert((int)Outgoing.size() == NumGuards + 1);
for (auto In : Incoming)
Updates.push_back({DominatorTree::Insert, In, FirstGuardBlock});
for (int i = 0; i != NumGuards - 1; ++i) {
Updates.push_back({DominatorTree::Insert, GuardBlocks[i], Outgoing[i]});
Updates.push_back(
{DominatorTree::Insert, GuardBlocks[i], GuardBlocks[i + 1]});
}
Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
Outgoing[NumGuards - 1]});
Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
Outgoing[NumGuards]});
DTU->applyUpdates(Updates);
}
for (auto I : DeletionCandidates) {
if (I->use_empty())
if (auto Inst = dyn_cast_or_null<Instruction>(I))
Inst->eraseFromParent();
}
return FirstGuardBlock;
}
void llvm::InvertBranch(BranchInst *PBI, IRBuilderBase &Builder) {
Value *NewCond = PBI->getCondition();
// If this is a "cmp" instruction, only used for branching (and nowhere
// else), then we can simply invert the predicate.
if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
CmpInst *CI = cast<CmpInst>(NewCond);
CI->setPredicate(CI->getInversePredicate());
} else
NewCond = Builder.CreateNot(NewCond, NewCond->getName() + ".not");
PBI->setCondition(NewCond);
PBI->swapSuccessors();
}
bool llvm::hasOnlySimpleTerminator(const Function &F) {
for (auto &BB : F) {
auto *Term = BB.getTerminator();
if (!(isa<ReturnInst>(Term) || isa<UnreachableInst>(Term) ||
isa<BranchInst>(Term)))
return false;
}
return true;
}
bool llvm::isPresplitCoroSuspendExitEdge(const BasicBlock &Src,
const BasicBlock &Dest) {
assert(Src.getParent() == Dest.getParent());
if (!Src.getParent()->isPresplitCoroutine())
return false;
if (auto *SW = dyn_cast<SwitchInst>(Src.getTerminator()))
if (auto *Intr = dyn_cast<IntrinsicInst>(SW->getCondition()))
return Intr->getIntrinsicID() == Intrinsic::coro_suspend &&
SW->getDefaultDest() == &Dest;
return false;
}
|