aboutsummaryrefslogtreecommitdiff
path: root/libjava/java/util/Random.java
diff options
context:
space:
mode:
Diffstat (limited to 'libjava/java/util/Random.java')
0 files changed, 0 insertions, 0 deletions
generated by cgit v1.1 at 2025-11-24 06:29:18 +0000
1' href='#n71'>71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989 7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181 8182 8183 8184 8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201 8202 8203 8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253 8254 8255 8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340 8341 8342 8343 8344 8345 8346 8347 8348 8349 8350 8351 8352 8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379 8380 8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426 8427 8428 8429 8430 8431 8432 8433 8434 8435 8436 8437 8438 8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474 8475 8476 8477 8478 8479 8480 8481 8482 8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494 8495 8496 8497 8498 8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619 8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714 8715 8716 8717 8718 8719 8720 8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802 8803 8804 8805 8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816 8817 8818 8819 8820 8821 8822 8823 8824 8825 8826 8827 8828 8829 8830 8831 8832 8833 8834 8835 8836 8837 8838 8839 8840 8841 8842 8843 8844 8845 8846 8847 8848 8849 8850 8851 8852 8853 8854 8855 8856 8857 8858 8859 8860 8861 8862 8863 8864 8865 8866 8867 8868 8869 8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883 8884 8885 8886 8887 8888 8889 8890 8891 8892 8893 8894 8895 8896 8897 8898 8899 8900 8901 8902 8903 8904 8905 8906 8907 8908 8909 8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930 8931 8932 8933 8934 8935 8936 8937 8938 8939 8940 8941 8942 8943 8944 8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956 8957 8958 8959 8960 8961 8962 8963 8964 8965 8966 8967 8968 8969 8970 8971 8972 8973 8974 8975 8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990 8991 8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002 9003 9004 9005 9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022 9023 9024 9025 9026 9027 9028 9029 9030 9031 9032 9033 9034 9035 9036 9037 9038 9039 9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051 9052 9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063 9064 9065 9066 9067 9068 9069 9070 9071 9072 9073 9074 9075 9076 9077 9078 9079 9080 9081 9082 9083 9084 9085 9086 9087 9088 9089 9090 9091 9092 9093 9094 9095 9096 9097 9098 9099 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109 9110 9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151 9152 9153 9154 9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172 9173 9174 9175 9176 9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201 9202 9203 9204 9205 9206 9207 9208 9209 9210 9211 9212 9213 9214 9215 9216 9217 9218 9219 9220 9221 9222 9223 9224 9225 9226 9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254 9255 9256 9257 9258 9259 9260 9261 9262 9263 9264 9265 9266 9267 9268 9269 9270 9271 9272 9273 9274 9275 9276 9277 9278 9279 9280 9281 9282 9283 9284 9285 9286 9287 9288 9289 9290 9291 9292 9293 9294 9295 9296 9297 9298 9299 9300 9301 9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317 9318 9319 9320 9321 9322 9323 9324 9325 9326 9327 9328 9329 9330 9331 9332 9333 9334 9335 9336 9337 9338 9339 9340 9341 9342 9343 9344 9345 9346 9347 9348 9349 9350 9351 9352 9353 9354 9355 9356 9357 9358 9359 9360 9361 9362 9363 9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412 9413 9414 9415 9416 9417 9418 9419 9420 9421 9422 9423 9424 9425 9426 9427 9428 9429 9430 9431 9432 9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449 9450 9451 9452 9453 9454 9455 9456 9457 9458 9459 9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530 9531 9532 9533 9534 9535 9536 9537 9538 9539 9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583 9584 9585 9586 9587 9588 9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608 9609 9610 9611 9612 9613 9614 9615 9616 9617 9618 9619 9620 9621 9622 9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634 9635 9636 9637 9638 9639 9640 9641 9642 9643 9644 9645 9646 9647 9648 9649 9650 9651 9652 9653 9654 9655 9656 9657 9658 9659 9660 9661 9662 9663 9664 9665 9666 9667 9668 9669 9670 9671 9672 9673 9674 9675 9676 9677 9678 9679 9680 9681 9682 9683 9684 9685 9686 9687 9688 9689 9690 9691 9692 9693 9694 9695 9696 9697 9698 9699 9700 9701 9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719 9720 9721 9722 9723 9724 9725 9726 9727 9728 9729 9730 9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746 9747 9748 9749 9750 9751 9752 9753 9754 9755 9756 9757 9758 9759 9760 9761 9762 9763 9764 9765 9766 9767 9768 9769 9770 9771 9772 9773 9774 9775 9776 9777 9778 9779 9780 9781 9782 9783 9784 9785 9786 9787 9788 9789 9790 9791 9792 9793 9794 9795 9796 9797 9798 9799 9800 9801 9802 9803 9804 9805 9806 9807 9808 9809 9810 9811 9812 9813 9814 9815 9816 9817 9818 9819 9820 9821 9822 9823 9824 9825 9826 9827 9828 9829 9830 9831 9832 9833 9834 9835 9836 9837 9838 9839 9840 9841 9842 9843 9844 9845 9846 9847 9848 9849 9850 9851 9852 9853 9854 9855 9856 9857 9858 9859 9860 9861 9862 9863 9864 9865 9866 9867 9868 9869 9870 9871 9872 9873 9874 9875 9876 9877 9878 9879 9880 9881 9882 9883 9884 9885 9886 9887 9888 9889 9890 9891 9892 9893 9894 9895 9896 9897 9898 9899 9900 9901 9902 9903 9904 9905 9906 9907 9908 9909 9910 9911 9912 9913 9914 9915 9916 9917 9918 9919 9920 9921 9922 9923 9924 9925 9926 9927 9928 9929 9930 9931 9932 9933 9934 9935 9936 9937 9938 9939 9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962 9963 9964 9965 9966 9967 9968 9969 9970 9971 9972 9973 9974 9975 9976 9977 9978 9979 9980 9981 9982 9983 9984 9985 9986 9987 9988 9989 9990 9991 9992 9993 9994 9995 9996 9997 9998 9999 10000 10001 10002 10003 10004 10005 10006 10007 10008 10009 10010 10011 10012 10013 10014 10015 10016 10017 10018 10019 10020 10021 10022 10023 10024 10025 10026 10027 10028 10029 10030 10031 10032 10033 10034 10035 10036 10037 10038 10039 10040 10041 10042 10043 10044 10045 10046 10047 10048 10049 10050 10051 10052 10053 10054 10055 10056 10057 10058 10059 10060 10061 10062 10063 10064 10065 10066 10067 10068 10069 10070 10071 10072 10073 10074 10075 10076 10077 10078 10079 10080 10081 10082 10083 10084 10085 10086 10087 10088 10089 10090 10091 10092 10093 10094 10095 10096 10097 10098 10099 10100 10101 10102 10103 10104 10105 10106 10107 10108 10109 10110 10111 10112 10113 10114 10115 10116 10117 10118 10119 10120 10121 10122 10123 10124 10125 10126 10127 10128 10129 10130 10131 10132 10133 10134 10135 10136 10137 10138 10139 10140 10141 10142 10143 10144 10145 10146 10147 10148 10149 10150 10151 10152 10153 10154 10155 10156 10157 10158 10159 10160 10161 10162 10163 10164 10165 10166 10167 10168 10169 10170 10171 10172 10173 10174 10175 10176 10177 10178 10179 10180 10181 10182 10183 10184 10185 10186 10187 10188 10189 10190 10191 10192 10193 10194 10195 10196 10197 10198 10199 10200 10201 10202 10203 10204 10205 10206 10207 10208 10209 10210 10211 10212 10213 10214 10215 10216 10217 10218 10219 10220 10221 10222 10223 10224 10225 10226 10227 10228 10229 10230 10231 10232 10233 10234 10235 10236 10237 10238 10239 10240 10241 10242 10243 10244 10245 10246 10247 10248 10249 10250 10251 10252 10253 10254 10255 10256 10257 10258 10259 10260 10261 10262 10263 10264 10265 10266 10267 10268 10269 10270 10271 10272 10273 10274 10275 10276 10277 10278 10279 10280 10281 10282 10283 10284 10285 10286 10287 10288 10289 10290 10291 10292 10293 10294 10295 10296 10297 10298 10299 10300 10301 10302 10303 10304 10305 10306 10307 10308 10309 10310 10311 10312 10313 10314 10315 10316 10317 10318 10319 10320 10321 10322 10323 10324 10325 10326 10327 10328 10329 10330 10331 10332 10333 10334 10335 10336 10337 10338 10339 10340 10341 10342 10343 10344 10345 10346 10347 10348 10349 10350 10351 10352 10353 10354 10355 10356 10357 10358 10359 10360 10361 10362 10363 10364 10365 10366 10367 10368 10369 10370 10371 10372 10373 10374 10375 10376 10377 10378 10379 10380 10381 10382 10383 10384 10385 10386 10387 10388 10389 10390 10391 10392 10393 10394 10395 10396 10397 10398 10399 10400 10401 10402 10403 10404 10405 10406 10407 10408 10409 10410 10411 10412 10413 10414 10415 10416 10417 10418 10419 10420 10421 10422 10423 10424 10425 10426 10427 10428 10429 10430 10431 10432 10433 10434 10435 10436 10437 10438 10439 10440 10441 10442 10443 10444 10445 10446 10447 10448 10449 10450 10451 10452 10453 10454 10455 10456 10457 10458 10459 10460 10461 10462 10463 10464 10465 10466 10467 10468 10469 10470 10471 10472 10473 10474 10475 10476 10477 10478 10479 10480 10481 10482 10483 10484 10485 10486 10487 10488 10489 10490 10491 10492 10493 10494 10495 10496 10497 10498 10499 10500 10501 10502 10503 10504 10505 10506 10507 10508 10509 10510 10511 10512 10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523 10524 10525 10526 10527 10528 10529 10530 10531 10532 10533 10534 10535 10536 10537 10538 10539 10540 10541 10542 10543 10544 10545 10546 10547 10548 10549 10550 10551 10552 10553 10554 10555 10556 10557 10558 10559 10560 10561 10562 10563 10564 10565 10566 10567 10568 10569 10570 10571 10572 10573 10574 10575 10576 10577 10578 10579 10580 10581 10582 10583 10584 10585 10586 10587 10588 10589 10590 10591 10592 10593 10594 10595 10596 10597 10598 10599 10600 10601 10602 10603 10604 10605 10606 10607 10608 10609 10610 10611 10612 10613 10614 10615 10616 10617 10618 10619 10620 10621 10622 10623 10624 10625 10626 10627 10628 10629 10630 10631 10632 10633 10634 10635 10636 10637 10638 10639 10640 10641 10642 10643 10644 10645 10646 10647 10648 10649 10650 10651 10652 10653 10654 10655 10656 10657 10658 10659 10660 10661 10662 10663 10664 10665 10666 10667 10668 10669 10670 10671 10672 10673 10674 10675 10676 10677 10678 10679 10680 10681 10682 10683 10684 10685 10686 10687 10688 10689 10690 10691 10692 10693 10694 10695 10696 10697 10698 10699 10700 10701 10702 10703 10704 10705 10706 10707 10708 10709 10710 10711 10712 10713 10714 10715 10716 10717 10718 10719 10720 10721 10722 10723 10724 10725 10726 10727 10728 10729 10730 10731 10732 10733 10734 10735 10736 10737 10738 10739 10740 10741 10742 10743 10744 10745 10746 10747 10748 10749 10750 10751 10752 10753 10754 10755 10756 10757 10758 10759 10760 10761 10762 10763 10764 10765 10766 10767 10768 10769 10770 10771 10772 10773 10774 10775 10776 10777 10778 10779 10780 10781 10782 10783 10784 10785 10786 10787 10788 10789 10790 10791 10792 10793 10794 10795 10796 10797 10798 10799 10800 10801 10802 10803 10804 10805 10806 10807 10808 10809 10810 10811 10812 10813 10814 10815 10816 10817 10818 10819 10820 10821 10822 10823 10824 10825 10826 10827 10828 10829 10830 10831 10832 10833 10834 10835 10836 10837 10838 10839 10840 10841 10842 10843 10844 10845 10846 10847 10848 10849 10850 10851 10852 10853 10854 10855 10856 10857 10858 10859 10860 10861 10862 10863 10864 10865 10866 10867 10868 10869 10870 10871 10872 10873 10874 10875 10876 10877 10878 10879 10880 10881 10882 10883 10884 10885 10886 10887 10888 10889 10890 10891 10892 10893 10894 10895 10896 10897 10898 10899 10900 10901 10902 10903 10904 10905 10906 10907 10908 10909 10910 10911 10912 10913 10914 10915 10916 10917 10918 10919 10920 10921 10922 10923 10924 10925 10926 10927 10928 10929 10930 10931 10932 10933 10934 10935 10936 10937 10938 10939 10940 10941 10942 10943 10944 10945 10946 10947 10948 10949 10950 10951 10952 10953 10954 10955 10956 10957 10958 10959 10960 10961 10962 10963 10964 10965 10966 10967 10968 10969 10970 10971 10972 10973 10974 10975 10976 10977 10978 10979 10980 10981 10982 10983 10984 10985 10986 10987 10988 10989 10990 10991 10992 10993 10994 10995 10996 10997 10998 10999 11000 11001 11002 11003 11004 11005 11006 11007 11008 11009 11010 11011 11012 11013 11014 11015 11016 11017 11018 11019 11020 11021 11022 11023 11024 11025 11026 11027 11028 11029 11030 11031 11032 11033 11034 11035 11036 11037 11038 11039 11040 11041 11042 11043 11044 11045 11046 11047 11048 11049 11050 11051 11052 11053 11054 11055 11056 11057 11058 11059 11060 11061 11062 11063 11064 11065 11066 11067 11068 11069 11070 11071 11072 11073 11074 11075 11076 11077 11078 11079 11080 11081 11082 11083 11084 11085 11086 11087 11088 11089 11090 11091 11092 11093 11094 11095 11096 11097 11098 11099 11100 11101 11102 11103 11104 11105 11106 11107 11108 11109 11110 11111 11112 11113 11114 11115 11116 11117 11118 11119 11120 11121 11122 11123 11124 11125 11126 11127 11128 11129 11130 11131 11132 11133 11134 11135 11136 11137 11138 11139 11140 11141 11142 11143 11144 11145 11146 11147 11148 11149 11150 11151 11152 11153 11154 11155 11156 11157 11158 11159 11160 11161 11162 11163 11164 11165 11166 11167 11168 11169 11170 11171 11172 11173 11174 11175 11176 11177 11178 11179 11180 11181 11182 11183 11184 11185 11186 11187 11188 11189 11190 11191 11192 11193 11194 11195 11196 11197 11198 11199 11200 11201 11202 11203 11204 11205 11206 11207 11208 11209 11210 11211 11212 11213 11214 11215 11216 11217 11218 11219 11220 11221 11222 11223 11224 11225 11226 11227 11228 11229 11230 11231 11232 11233 11234 11235 11236 11237 11238 11239 11240 11241 11242 11243 11244 11245 11246 11247 11248 11249 11250 11251 11252 11253 11254 11255 11256 11257 11258 11259 11260 11261 11262 11263 11264 11265 11266 11267 11268 11269 11270 11271 11272 11273 11274 11275 11276 11277 11278 11279 11280 11281 11282 11283 11284 11285 11286 11287 11288 11289 11290 11291 11292 11293 11294 11295 11296 11297 11298 11299 11300 11301 11302 11303 11304 11305 11306 11307 11308 11309 11310 11311 11312 11313 11314 11315 11316 11317 11318 11319 11320 11321 11322 11323 11324 11325 11326 11327 11328 11329 11330 11331 11332 11333 11334 11335 11336 11337 11338 11339 11340 11341 11342 11343 11344 11345 11346 11347 11348 11349 11350 11351 11352 11353 11354 11355 11356 11357 11358 11359 11360 11361 11362 11363 11364 11365 11366 11367 11368 11369 11370 11371 11372 11373 11374 11375 11376 11377 11378 11379 11380 11381 11382 11383 11384 11385 11386 11387 11388 11389 11390 11391 11392 11393 11394 11395 11396 11397 11398 11399 11400 11401 11402 11403 11404 11405 11406 11407 11408 11409 11410 11411 11412 11413 11414 11415 11416 11417 11418 11419 11420 11421 11422 11423 11424 11425 11426 11427 11428 11429 11430 11431 11432 11433 11434 11435 11436 11437 11438 11439 11440 11441 11442 11443 11444 11445 11446 11447 11448 11449 11450 11451 11452 11453 11454 11455 11456 11457 11458 11459 11460 11461 11462 11463 11464 11465 11466 11467 11468 11469 11470 11471 11472 11473 11474 11475 11476 11477 11478 11479 11480 11481 11482 11483 11484 11485 11486 11487 11488 11489 11490 11491 11492 11493 11494 11495 11496 11497 11498 11499 11500 11501 11502 11503 11504 11505 11506 11507 11508 11509 11510 11511 11512 11513 11514 11515 11516 11517 11518 11519 11520 11521 11522 11523 11524 11525 11526 11527 11528 11529 11530 11531 11532 11533 11534 11535 11536 11537 11538 11539 11540 11541 11542 11543 11544 11545 11546 11547 11548 11549 11550 11551 11552 11553 11554 11555 11556 11557 11558 11559 11560 11561 11562 11563 11564 11565 11566 11567 11568 11569 11570 11571 11572 11573 11574 11575 11576 11577 11578 11579 11580 11581 11582 11583 11584 11585 11586 11587 11588 11589 11590 11591 11592 11593 11594 11595 11596 11597 11598 11599 11600 11601 11602 11603 11604 11605 11606 11607 11608 11609 11610 11611 11612 11613 11614 11615 11616 11617 11618 11619 11620 11621 11622 11623 11624 11625 11626 11627 11628 11629 11630 11631 11632 11633 11634 11635 11636 11637 11638 11639 11640 11641 11642 11643 11644 11645 11646 11647 11648 11649 11650 11651 11652 11653 11654 11655 11656 11657 11658 11659 11660 11661 11662 11663 11664 11665 11666 11667 11668 11669 11670 11671 11672 11673 11674 11675 11676 11677 11678 11679 11680 11681 11682 11683 11684 11685 11686 11687 11688 11689 11690 11691 11692 11693 11694 11695 11696 11697 11698 11699 11700 11701 11702 11703 11704 11705 11706 11707 11708 11709 11710 11711 11712 11713 11714 11715 11716 11717 11718 11719 11720 11721 11722 11723 11724 11725 11726 11727 11728 11729 11730 11731 11732 11733 11734 11735 11736 11737 11738 11739 11740 11741 11742 11743 11744 11745 11746 11747 11748 11749 11750 11751 11752 11753 11754 11755 11756 11757 11758 11759 11760 11761 11762 11763 11764 11765 11766 11767 11768 11769 11770 11771 11772 11773 11774 11775 11776 11777 11778 11779 11780 11781 11782 11783 11784 11785 11786 11787 11788 11789 11790 11791 11792 11793 11794 11795 11796 11797 11798 11799 11800 11801 11802 11803 11804 11805 11806 11807 11808 11809 11810 11811 11812 11813 11814 11815 11816 11817 11818 11819 11820 11821 11822 11823 11824 11825 11826 11827 11828 11829 11830 11831 11832 11833 11834 11835 11836 11837 11838 11839 11840 11841 11842 11843 11844 11845 11846 11847 11848 11849 11850 11851 11852 11853 11854 11855 11856 11857 11858 11859 11860 11861 11862 11863 11864 11865 11866 11867 11868 11869 11870 11871 11872 11873 11874 11875 11876 11877 11878 11879 11880 11881 11882 11883 11884 11885 11886 11887 11888 11889 11890 11891 11892 11893 11894 11895 11896 11897 11898 11899 11900 11901 11902 11903 11904 11905 11906 11907 11908 11909 11910 11911 11912 11913 11914 11915 11916 11917 11918 11919 11920 11921 11922 11923 11924 11925 11926 11927 11928 11929 11930 11931 11932 11933 11934 11935 11936 11937 11938 11939 11940 11941 11942 11943 11944 11945 11946 11947 11948 11949 11950 11951 11952 11953 11954 11955 11956 11957 11958 11959 11960 11961 11962 11963 11964 11965 11966 11967 11968 11969 11970 11971 11972 11973 11974 11975 11976 11977 11978 11979 11980 11981 11982 11983 11984 11985 11986 11987 11988 11989 11990 11991 11992 11993 11994 11995 11996 11997 11998 11999 12000 12001 12002 12003 12004 12005 12006 12007 12008 12009 12010 12011 12012 12013 12014 12015 12016 12017 12018 12019 12020 12021 12022 12023 12024 12025 12026 12027 12028 12029 12030 12031 12032 12033 12034 12035 12036 12037 12038 12039 12040 12041 12042 12043 12044 12045 12046 12047 12048 12049 12050 12051 12052 12053 12054 12055 12056 12057 12058 12059 12060 12061 12062 12063 12064 12065 12066 12067 12068 12069 12070 12071 12072 12073 12074 12075 12076 12077 12078 12079 12080 12081 12082 12083 12084 12085 12086 12087 12088 12089 12090 12091 12092 12093 12094 12095 12096 12097 12098 12099 12100 12101 12102 12103 12104 12105 12106 12107 12108 12109 12110 12111 12112 12113 12114 12115 12116 12117 12118 12119 12120 12121 12122 12123 12124 12125 12126 12127 12128 12129 12130 12131 12132 12133 12134 12135 12136 12137 12138 12139 12140 12141 12142 12143 12144 12145 12146 12147 12148 12149 12150 12151 12152 12153 12154 12155 12156 12157 12158 12159 12160 12161 12162 12163 12164 12165 12166 12167 12168 12169 12170 12171 12172 12173 12174 12175 12176 12177 12178 12179 12180 12181 12182 12183 12184 12185 12186 12187 12188 12189 12190 12191 12192 12193 12194 12195 12196 12197 12198 12199 12200 12201 12202 12203 12204 12205 12206 12207 12208 12209 12210 12211 12212 12213 12214 12215 12216 12217 12218 12219 12220 12221 12222 12223 12224 12225 12226 12227 12228 12229 12230 12231 12232 12233 12234 12235 12236 12237 12238 12239 12240 12241 12242 12243 12244 12245 12246 12247 12248 12249 12250 12251 12252 12253 12254 12255 12256 12257 12258 12259 12260 12261 12262 12263 12264 12265 12266 12267 12268 12269 12270 12271 12272 12273 12274 12275 12276 12277 12278 12279 12280 12281 12282 12283 12284 12285 12286 12287 12288 12289 12290 12291 12292 12293 12294 12295 12296 12297 12298 12299 12300 12301 12302 12303 12304 12305 12306 12307 12308 12309 12310 12311 12312 12313 12314 12315 12316 12317 12318 12319 12320 12321 12322 12323 12324 12325 12326 12327 12328 12329 12330 12331 12332 12333 12334 12335 12336 12337 12338 12339 12340 12341 12342 12343 12344 12345 12346 12347 12348 12349 12350 12351 12352 12353 12354 12355 12356 12357 12358 12359 12360 12361 12362 12363 12364 12365 12366 12367 12368 12369 12370 12371 12372 12373 12374 12375 12376 12377 12378 12379 12380 12381 12382 12383 12384 12385 12386 12387 12388 12389 12390 12391 12392 12393 12394 12395 12396 12397 12398 12399 12400 12401 12402 12403 12404 12405 12406 12407 12408 12409 12410 12411 12412 12413 12414 12415 12416 12417 12418 12419 12420 12421 12422 12423 12424 12425 12426 12427 12428 12429 12430 12431 12432 12433 12434 12435 12436 12437 12438 12439 12440 12441 12442 12443 12444 12445 12446 12447 12448 12449 12450 12451 12452 12453 12454 12455 12456 12457 12458 12459 12460 12461 12462 12463 12464 12465 12466 12467 12468 12469 12470 12471 12472 12473 12474 12475 12476 12477 12478 12479 12480 12481 12482 12483 12484 12485 12486 12487 12488 12489 12490 12491 12492 12493 12494 12495 12496 12497 12498 12499 12500 12501 12502 12503 12504 12505 12506 12507 12508 12509 12510 12511 12512 12513 12514 12515 12516 12517 12518 12519 12520 12521 12522 12523 12524 12525 12526 12527 12528 12529 12530 12531 12532 12533 12534 12535 12536 12537 12538 12539 12540 12541 12542 12543 12544 12545 12546 12547 12548 12549 12550 12551 12552 12553 12554 12555 12556 12557 12558 12559 12560 12561 12562 12563 12564 12565 12566 12567 12568 12569 12570 12571 12572 12573 12574 12575 12576 12577 12578 12579 12580 12581 12582 12583 12584 12585 12586 12587 12588 12589 12590 12591 12592 12593 12594 12595 12596 12597 12598 12599 12600 12601 12602 12603 12604 12605 12606 12607 12608 12609 12610 12611 12612 12613 12614 12615 12616 12617 12618 12619 12620 12621 12622 12623 12624 12625 12626 12627 12628 12629 12630 12631 12632 12633 12634 12635 12636 12637 12638 12639 12640 12641 12642 12643 12644 12645 12646 12647 12648 12649 12650 12651 12652 12653 12654 12655 12656 12657 12658 12659 12660 12661 12662 12663 12664 12665 12666 12667 12668 12669 12670 12671 12672 12673 12674 12675 12676 12677 12678 12679 12680 12681 12682 12683 12684 12685 12686 12687 12688 12689 12690 12691 12692 12693 12694 12695 12696 12697 12698 12699 12700 12701 12702 12703 12704 12705 12706 12707 12708 12709 12710 12711 12712 12713 12714 12715 12716 12717 12718 12719 12720 12721 12722 12723 12724 12725 12726 12727 12728 12729 12730 12731 12732 12733 12734 12735 12736 12737 12738 12739 12740 12741 12742 12743 12744 12745 12746 12747 12748 12749 12750 12751 12752 12753 12754 12755 12756 12757 12758 12759 12760 12761 12762 12763 12764 12765 12766 12767 12768 12769 12770 12771 12772 12773 12774 12775 12776 12777 12778 12779 12780 12781 12782 12783 12784 12785 12786 12787 12788 12789 12790 12791 12792 12793 12794 12795 12796 12797 12798 12799 12800 12801 12802 12803 12804 12805 12806 12807 12808 12809 12810 12811 12812 12813 12814 12815 12816 12817 12818 12819 12820 12821 12822 12823 12824 12825 12826 12827 12828 12829 12830 12831 12832 12833 12834 12835 12836 12837 12838 12839 12840 12841 12842 12843 12844 12845 12846 12847 12848 12849 12850 12851 12852 12853 12854 12855 12856 12857 12858 12859 12860 12861 12862 12863 12864 12865 12866 12867 12868 12869 12870 12871 12872 12873 12874 12875 12876 12877 12878 12879 12880 12881 12882 12883 12884 12885 12886 12887 12888 12889 12890 12891 12892 12893 12894 12895 12896 12897 12898 12899 12900 12901 12902 12903 12904 12905 12906 12907 12908 12909 12910 12911 12912 12913 12914 12915 12916 12917 12918 12919 12920 12921 12922 12923 12924 12925 12926 12927 12928 12929 12930 12931 12932 12933 12934 12935 12936 12937 12938 12939 12940 12941 12942 12943 12944 12945 12946 12947 12948 12949 12950 12951 12952 12953 12954 12955 12956 12957 12958 12959 12960 12961 12962 12963 12964 12965 12966 12967 12968 12969 12970 12971 12972 12973 12974 12975 12976 12977 12978 12979 12980 12981 12982 12983 12984 12985 12986 12987 12988 12989 12990 12991 12992 12993 12994 12995 12996 12997 12998 12999 13000 13001 13002 13003 13004 13005 13006 13007 13008 13009 13010 13011 13012 13013 13014 13015 13016 13017 13018 13019 13020 13021 13022 13023 13024 13025 13026 13027 13028 13029 13030 13031 13032 13033 13034 13035 13036 13037 13038 13039 13040 13041 13042 13043 13044 13045 13046 13047 13048 13049 13050 13051 13052 13053 13054 13055 13056 13057 13058 13059 13060 13061 13062 13063 13064 13065 13066 13067 13068 13069 13070 13071 13072 13073 13074 13075 13076 13077 13078 13079 13080 13081 13082 13083 13084 13085 13086 13087 13088 13089 13090 13091 13092 13093 13094 13095 13096 13097 13098 13099 13100 13101 13102 13103 13104 13105 13106 13107 13108 13109 13110 13111 13112 13113 13114 13115 13116 13117 13118 13119 13120 13121 13122 13123 13124 13125 13126 13127 13128 13129 13130 13131 13132 13133 13134 13135 13136 13137 13138 13139 13140 13141 13142 13143 13144 13145 13146 13147 13148 13149 13150 13151 13152 13153 13154 13155 13156 13157 13158 13159 13160 13161 13162 13163 13164 13165 13166 13167 13168 13169 13170 13171 13172 13173 13174 13175 13176 13177 13178 13179 13180 13181 13182 13183 13184 13185 13186 13187 13188 13189 13190 13191 13192 13193 13194 13195 13196 13197 13198 13199 13200 13201 13202 13203 13204 13205 13206 13207 13208 13209 13210 13211 13212 13213 13214 13215 13216 13217 13218 13219 13220 13221 13222 13223 13224 13225 13226 13227 13228 13229 13230 13231 13232 13233 13234 13235 13236 13237 13238 13239 13240 13241 13242 13243 13244 13245 13246 13247 13248 13249 13250 13251 13252 13253 13254 13255 13256 13257 13258 13259 13260 13261 13262 13263 13264 13265 13266 13267 13268 13269 13270 13271 13272 13273 13274 13275 13276 13277 13278 13279 13280 13281 13282 13283 13284 13285 13286 13287 13288 13289 13290 13291 13292 13293 13294 13295 13296 13297 13298 13299 13300 13301 13302 13303 13304 13305 13306 13307 13308 13309 13310 13311 13312 13313 13314 13315 13316 13317 13318 13319 13320 13321 13322 13323 13324 13325 13326 13327 13328 13329 13330 13331 13332 13333 13334 13335 13336 13337 13338 13339 13340 13341 13342 13343 13344 13345 13346 13347 13348 13349 13350 13351 13352 13353 13354 13355 13356 13357 13358 13359 13360 13361 13362 13363 13364 13365 13366 13367 13368 13369 13370 13371 13372 13373 13374 13375 13376 13377 13378 13379 13380 13381 13382 13383 13384 13385 13386 13387 13388 13389 13390 13391 13392 13393 13394 13395 13396 13397 13398 13399 13400 13401 13402 13403 13404 13405 13406 13407 13408 13409 13410 13411 13412 13413 13414 13415 13416 13417 13418 13419 13420 13421 13422 13423 13424 13425 13426 13427 13428 13429 13430 13431 13432 13433 13434 13435 13436 13437 13438 13439 13440 13441 13442 13443 13444 13445 13446 13447 13448 13449 13450 13451 13452 13453 13454 13455 13456 13457 13458 13459 13460 13461 13462 13463 13464 13465 13466 13467 13468 13469 13470 13471 13472 13473 13474 13475 13476 13477 13478 13479 13480 13481 13482 13483 13484 13485 13486 13487 13488 13489 13490 13491 13492 13493 13494 13495 13496 13497 13498 13499 13500 13501 13502 13503 13504 13505 13506 13507 13508 13509 13510 13511 13512 13513 13514 13515 13516 13517 13518 13519 13520 13521 13522 13523 13524 13525 13526 13527 13528 13529 13530 13531 13532 13533 13534 13535 13536 13537 13538 13539 13540 13541 13542 13543 13544 13545 13546 13547 13548 13549 13550 13551 13552 13553 13554 13555 13556 13557 13558 13559 13560 13561 13562 13563 13564 13565 13566 13567 13568 13569 13570 13571 13572 13573 13574 13575 13576 13577 13578 13579 13580 13581 13582 13583 13584 13585 13586 13587 13588 13589 13590 13591 13592 13593 13594 13595 13596 13597 13598 13599 13600 13601 13602 13603 13604 13605 13606 13607 13608 13609 13610 13611 13612 13613 13614 13615 13616 13617 13618 13619 13620 13621 13622 13623 13624 13625 13626 13627 13628 13629 13630 13631 13632 13633 13634 13635 13636 13637 13638 13639 13640 13641 13642 13643 13644 13645 13646 13647 13648 13649 13650 13651 13652 13653 13654 13655 13656 13657 13658 13659 13660 13661 13662 13663 13664 13665 13666 13667 13668 13669 13670 13671 13672 13673 13674 13675 13676 13677 13678 13679 13680 13681 13682 13683 13684 13685 13686 13687 13688 13689 13690 13691 13692 13693 13694 13695 13696 13697 13698 13699 13700 13701 13702 13703 13704 13705 13706 13707 13708 13709 13710 13711 13712 13713 13714 13715 13716 13717 13718 13719 13720 13721 13722 13723 13724 13725 13726 13727 13728 13729 13730 13731 13732 13733 13734 13735 13736 13737 13738 13739 13740 13741 13742 13743 13744 13745 13746 13747 13748 13749 13750 13751 13752 13753 13754 13755 13756 13757 13758 13759 13760 13761 13762 13763 13764 13765 13766 13767 13768 13769 13770 13771 13772 13773 13774 13775 13776 13777 13778 13779 13780 13781 13782 13783 13784 13785 13786 13787 13788 13789 13790 13791 13792 13793 13794 13795 13796 13797 13798 13799 13800 13801 13802 13803 13804 13805 13806 13807 13808 13809 13810 13811 13812 13813 13814 13815 13816 13817 13818 13819 13820 13821 13822 13823 13824 13825 13826 13827 13828 13829 13830 13831 13832 13833 13834 13835 13836 13837 13838 13839 13840 13841 13842 13843 13844 13845 13846 13847 13848 13849 13850 13851 13852 13853 13854 13855 13856 13857 13858 13859 13860 13861 13862 13863 13864 13865 13866 13867 13868 13869 13870 13871 13872 13873 13874 13875 13876 13877 13878 13879 13880 13881 13882 13883 13884 13885 13886 13887 13888 13889 13890 13891 13892 13893 13894 13895 13896 13897 13898 13899 13900 13901 13902 13903 13904 13905 13906 13907 13908 13909 13910 13911 13912 13913 13914 13915 13916 13917 13918 13919 13920 13921 13922 13923 13924 13925 13926 13927 13928 13929 13930 13931 13932 13933 13934 13935 13936 13937 13938 13939 13940 13941 13942 13943 13944 13945 13946 13947 13948 13949 13950 13951 13952 13953 13954 13955 13956 13957 13958 13959 13960 13961 13962 13963 13964 13965 13966 13967 13968 13969 13970 13971 13972 13973 13974 13975 13976 13977 13978 13979 13980 13981 13982 13983 13984 13985 13986 13987 13988 13989 13990 13991 13992 13993 13994 13995 13996 13997 13998 13999 14000 14001 14002 14003 14004 14005 14006 14007 14008 14009 14010 14011 14012 14013 14014 14015 14016 14017 14018 14019 14020 14021 14022 14023 14024 14025 14026 14027 14028 14029 14030 14031 14032 14033 14034 14035 14036 14037 14038 14039 14040 14041 14042 14043 14044 14045 14046 14047 14048 14049 14050 14051 14052 14053 14054 14055 14056 14057 14058 14059 14060 14061 14062 14063 14064 14065 14066 14067 14068 14069 14070 14071 14072 14073 14074 14075 14076 14077 14078 14079 14080 14081 14082 14083 14084 14085 14086 14087 14088 14089 14090 14091 14092 14093 14094 14095 14096 14097 14098 14099 14100 14101 14102 14103 14104 14105 14106 14107 14108 14109 14110 14111 14112 14113 14114 14115 14116 14117 14118 14119 14120 14121 14122 14123 14124 14125 14126 14127 14128 14129 14130 14131 14132 14133 14134 14135 14136 14137 14138 14139 14140 14141 14142 14143 14144 14145 14146 14147 14148 14149 14150 14151 14152 14153 14154 14155 14156 14157 14158 14159 14160 14161 14162 14163 14164 14165 14166 14167 14168 14169 14170 14171 14172 14173 14174 14175 14176 14177 14178 14179 14180 14181 14182 14183 14184 14185 14186 14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 14210 14211 14212 14213 14214 14215 14216 14217 14218 14219 14220 14221 14222 14223 14224 14225 14226 14227 14228 14229 14230 14231 14232 14233 14234 14235 14236 14237 14238 14239 14240 14241 14242 14243 14244 14245 14246 14247 14248 14249 14250 14251 14252 14253 14254 14255 14256 14257 14258 14259 14260 14261 14262 14263 14264 14265 14266 14267 14268 14269 14270 14271 14272 14273 14274 14275 14276 14277 14278 14279 14280 14281 14282 14283 14284 14285 14286 14287 14288 14289 14290 14291 14292 14293 14294 14295 14296 14297 14298 14299 14300 14301 14302 14303 14304 14305 14306 14307 14308 14309 14310 14311 14312 14313 14314 14315 14316 14317 14318 14319 14320 14321 14322 14323 14324 14325 14326 14327 14328 14329 14330 14331 14332 14333 14334 14335 14336 14337 14338 14339 14340 14341 14342 14343 14344 14345 14346 14347 14348 14349 14350 14351 14352 14353 14354 14355 14356 14357 14358 14359 14360 14361 14362 14363 14364 14365 14366 14367 14368 14369 14370 14371 14372 14373 14374 14375 14376 14377 14378 14379 14380 14381 14382 14383 14384 14385 14386 14387 14388 14389 14390 14391 14392 14393 14394 14395 14396 14397 14398 14399 14400 14401 14402 14403 14404 14405 14406 14407 14408 14409 14410 14411 14412 14413 14414 14415 14416 14417 14418 14419 14420 14421 14422 14423 14424 14425 14426 14427 14428 14429 14430 14431 14432 14433 14434 14435 14436 14437 14438 14439 14440 14441 14442 14443 14444 14445 14446 14447 14448 14449 14450 14451 14452 14453 14454 14455 14456 14457 14458 14459 14460 14461 14462 14463 14464 14465 14466 14467 14468 14469 14470 14471 14472 14473 14474 14475 14476 14477 14478 14479 14480 14481 14482 14483 14484 14485 14486 14487 14488 14489 14490 14491 14492 14493 14494 14495 14496 14497 14498 14499 14500 14501 14502 14503 14504 14505 14506 14507 14508 14509 14510 14511 14512 14513 14514 14515 14516 14517 14518 14519 14520 14521 14522 14523 14524 14525 14526 14527 14528 14529 14530 14531 14532 14533 14534 14535 14536 14537 14538 14539 14540 14541 14542 14543 14544 14545 14546 14547 14548 14549 14550 14551 14552 14553 14554 14555 14556 14557 14558 14559 14560 14561 14562 14563 14564 14565 14566 14567 14568 14569 14570 14571 14572 14573 14574 14575 14576 14577 14578 14579 14580 14581 14582 14583 14584 14585 14586 14587 14588 14589 14590 14591 14592 14593 14594 14595 14596 14597 14598 14599 14600 14601 14602 14603 14604 14605 14606 14607 14608 14609 14610 14611 14612 14613 14614 14615 14616 14617 14618 14619 14620 14621 14622 14623 14624 14625 14626 14627 14628 14629 14630 14631 14632 14633 14634 14635 14636 14637 14638 14639 14640 14641 14642 14643 14644 14645 14646 14647 14648 14649 14650 14651 14652 14653 14654 14655 14656 14657 14658 14659 14660 14661 14662 14663 14664 14665 14666 14667 14668 14669 14670 14671 14672 14673 14674 14675 14676 14677 14678 14679 14680 14681 14682 14683 14684 14685 14686 14687 14688 14689 14690 14691 14692 14693 14694 14695 14696 14697 14698 14699 14700 14701 14702 14703 14704 14705 14706 14707 14708 14709 14710 14711 14712 14713 14714 14715 14716 14717 14718 14719 14720 14721 14722 14723 14724 14725 14726 14727 14728 14729 14730 14731 14732 14733 14734 14735 14736 14737 14738 14739 14740 14741 14742 14743 14744 14745 14746 14747 14748 14749 14750 14751 14752 14753 14754 14755 14756 14757 14758 14759 14760 14761 14762 14763 14764 14765 14766 14767 14768 14769 14770 14771 14772 14773 14774 14775 14776 14777 14778 14779 14780 14781 14782 14783 14784 14785 14786 14787 14788 14789 14790 14791 14792 14793 14794 14795 14796 14797 14798 14799 14800 14801 14802 14803 14804 14805 14806 14807 14808 14809 14810 14811 14812 14813 14814 14815 14816 14817 14818 14819 14820 14821 14822 14823 14824 14825 14826 14827 14828 14829 14830 14831 14832 14833 14834 14835 14836 14837 14838 14839 14840 14841 14842 14843 14844 14845 14846 14847 14848 14849 14850 14851 14852 14853 14854 14855 14856 14857 14858 14859 14860 14861 14862 14863 14864 14865 14866 14867 14868 14869 14870 14871 14872 14873 14874 14875 14876 14877 14878 14879 14880 14881 14882 14883 14884 14885 14886 14887 14888 14889 14890 14891 14892 14893 14894 14895 14896 14897 14898 14899 14900 14901 14902 14903 14904 14905 14906 14907 14908 14909 14910 14911 14912 14913 14914 14915 14916 14917 14918 14919 14920 14921 14922 14923 14924 14925 14926 14927 14928 14929 14930 14931 14932 14933 14934 14935 14936 14937 14938 14939 14940 14941 14942 14943 14944 14945 14946 14947 14948 14949 14950 14951 14952 14953 14954 14955 14956 14957 14958 14959 14960 14961 14962 14963 14964 14965 14966 14967 14968 14969 14970 14971 14972 14973 14974 14975 14976 14977 14978 14979 14980 14981 14982 14983 14984 14985 14986 14987 14988 14989 14990 14991 14992 14993 14994 14995 14996 14997 14998 14999 15000 15001 15002 15003 15004 15005 15006 15007 15008 15009 15010 15011 15012 15013 15014 15015 15016 15017 15018 15019 15020 15021 15022 15023 15024 15025 15026 15027 15028 15029 15030 15031 15032 15033 15034 15035 15036 15037 15038 15039 15040 15041 15042 15043 15044 15045 15046 15047 15048 15049 15050 15051 15052 15053 15054 15055 15056 15057 15058 15059 15060 15061 15062 15063 15064 15065 15066 15067 15068 15069 15070 15071 15072 15073 15074 15075 15076 15077 15078 15079 15080 15081 15082 15083 15084 15085 15086 15087 15088 15089 15090 15091 15092 15093 15094 15095 15096 15097 15098 15099 15100 15101 15102 15103 15104 15105 15106 15107 15108 15109 15110 15111 15112 15113 15114 15115 15116 15117 15118 15119 15120 15121 15122 15123 15124 15125 15126 15127 15128 15129 15130 15131 15132 15133 15134 15135 15136 15137 15138 15139 15140 15141 15142 15143 15144 15145 15146 15147 15148 15149 15150 15151 15152 15153 15154 15155 15156 15157 15158 15159 15160 15161 15162 15163 15164 15165 15166 15167 15168 15169 15170 15171 15172 15173 15174 15175 15176 15177 15178 15179 15180 15181 15182 15183 15184 15185 15186 15187 15188 15189 15190 15191 15192 15193 15194 15195 15196 15197 15198 15199 15200 15201 15202 15203 15204 15205 15206 15207 15208 15209 15210 15211 15212 15213 15214 15215 15216 15217 15218 15219 15220 15221 15222 15223 15224 15225 15226 15227 15228 15229 15230 15231 15232 15233 15234 15235 15236 15237 15238 15239 15240 15241 15242 15243 15244 15245 15246 15247 15248 15249 15250 15251 15252 15253 15254 15255 15256 15257 15258 15259 15260 15261 15262 15263 15264 15265 15266 15267 15268 15269 15270 15271 15272 15273 15274 15275 15276 15277 15278 15279 15280 15281 15282 15283 15284 15285 15286 15287 15288 15289 15290 15291 15292 15293 15294 15295 15296 15297 15298 15299 15300 15301 15302 15303 15304 15305 15306 15307 15308 15309 15310 15311 15312 15313 15314 15315 15316 15317 15318 15319 15320 15321 15322 15323 15324 15325 15326 15327 15328 15329 15330 15331 15332 15333 15334 15335 15336 15337 15338 15339 15340 15341 15342 15343 15344 15345 15346 15347 15348 15349 15350 15351 15352 15353 15354 15355 15356 15357 15358 15359 15360 15361 15362 15363 15364 15365 15366 15367 15368 15369 15370 15371 15372 15373 15374 15375 15376 15377 15378 15379 15380 15381 15382 15383 15384 15385 15386 15387 15388 15389 15390 15391 15392 15393 15394 15395 15396 15397 15398 15399 15400 15401 15402 15403 15404 15405 15406 15407 15408 15409 15410 15411 15412 15413 15414 15415 15416 15417 15418 15419 15420 15421 15422 15423 15424 15425 15426 15427 15428 15429 15430 15431 15432 15433 15434 15435 15436 15437 15438 15439 15440 15441 15442 15443 15444 15445 15446 15447 15448 15449 15450 15451 15452 15453 15454 15455 15456 15457 15458 15459 15460 15461 15462 15463 15464 15465 15466 15467 15468 15469 15470 15471 15472 15473 15474 15475 15476 15477 15478 15479 15480 15481 15482 15483 15484 15485 15486 15487 15488 15489 15490 15491 15492 15493 15494 15495 15496 15497 15498 15499 15500 15501 15502 15503 15504 15505 15506 15507 15508 15509 15510 15511 15512 15513 15514 15515 15516 15517 15518 15519 15520 15521 15522 15523 15524 15525 15526 15527 15528 15529 15530 15531 15532 15533 15534 15535 15536 15537 15538 15539 15540 15541 15542 15543 15544 15545 15546 15547 15548 15549 15550 15551 15552 15553 15554 15555 15556 15557 15558 15559 15560 15561 15562 15563 15564 15565 15566 15567 15568 15569 15570 15571 15572 15573 15574 15575 15576 15577 15578 15579 15580 15581 15582 15583 15584 15585 15586 15587 15588 15589 15590 15591 15592 15593 15594 15595 15596 15597 15598 15599 15600 15601 15602 15603 15604 15605 15606 15607 15608 15609 15610 15611 15612 15613 15614 15615 15616 15617 15618 15619 15620 15621 15622 15623 15624 15625 15626 15627 15628 15629 15630 15631 15632 15633 15634 15635 15636 15637 15638 15639 15640 15641 15642 15643 15644 15645 15646 15647 15648 15649 15650 15651 15652 15653 15654 15655 15656 15657 15658 15659 15660 15661 15662 15663 15664 15665 15666 15667 15668 15669 15670 15671 15672 15673 15674 15675 15676 15677 15678 15679 15680 15681 15682 15683 15684 15685 15686 15687 15688 15689 15690 15691 15692 15693 15694 15695 15696 15697 15698 15699 15700 15701 15702 15703 15704 15705 15706 15707 15708 15709 15710 15711 15712 15713 15714 15715 15716 15717 15718 15719 15720 15721 15722 15723 15724 15725 15726 15727 15728 15729 15730 15731 15732 15733 15734 15735 15736 15737 15738 15739 15740 15741 15742 15743 15744 15745 15746 15747 15748 15749 15750 15751 15752 15753 15754 15755 15756 15757 15758 15759 15760 15761 15762 15763 15764 15765 15766 15767 15768 15769 15770 15771 15772 15773 15774 15775 15776 15777 15778 15779 15780 15781 15782 15783 15784 15785 15786 15787 15788 15789 15790 15791 15792 15793 15794 15795 15796 15797 15798 15799 15800 15801 15802 15803 15804 15805 15806 15807 15808 15809 15810 15811 15812 15813 15814 15815 15816 15817 15818 15819 15820 15821 15822 15823 15824 15825 15826 15827 15828 15829 15830 15831 15832 15833 15834 15835 15836 15837 15838 15839 15840 15841 15842 15843 15844 15845 15846 15847 15848 15849 15850 15851 15852 15853 15854 15855 15856 15857 15858 15859 15860 15861 15862 15863 15864 15865 15866 15867 15868 15869 15870 15871 15872 15873 15874 15875 15876 15877 15878 15879 15880 15881 15882 15883 15884 15885 15886 15887 15888 15889 15890 15891 15892 15893 15894 15895 15896 15897 15898 15899 15900 15901 15902 15903 15904 15905 15906 15907 15908 15909 15910 15911 15912 15913 15914 15915 15916 15917 15918 15919 15920 15921 15922 15923 15924 15925 15926 15927 15928 15929 15930 15931 15932 15933 15934 15935 15936 15937 15938 15939 15940 15941 15942 15943 15944 15945 15946 15947 15948 15949 15950 15951 15952 15953 15954 15955 15956 15957 15958 15959 15960 15961 15962 15963 15964 15965 15966 15967 15968 15969 15970 15971 15972 15973 15974 15975 15976 15977 15978 15979 15980 15981 15982 15983 15984 15985 15986 15987 15988 15989 15990 15991 15992 15993 15994 15995 15996 15997 15998 15999 16000 16001 16002 16003 16004 16005 16006 16007 16008 16009 16010 16011 16012 16013 16014 16015 16016 16017 16018 16019 16020 16021 16022 16023 16024 16025 16026 16027 16028 16029 16030 16031 16032 16033 16034 16035 16036 16037 16038 16039 16040 16041 16042 16043 16044 16045 16046 16047 16048 16049 16050 16051 16052 16053 16054 16055 16056 16057 16058 16059 16060 16061 16062 16063 16064 16065 16066 16067 16068 16069 16070 16071 16072 16073 16074 16075 16076 16077 16078 16079 16080 16081 16082 16083 16084 16085 16086 16087 16088 16089 16090 16091 16092 16093 16094 16095 16096 16097 16098 16099 16100 16101 16102 16103 16104 16105 16106 16107 16108 16109 16110 16111 16112 16113 16114 16115 16116 16117 16118 16119 16120 16121 16122 16123 16124 16125 16126 16127 16128 16129 16130 16131 16132 16133 16134 16135 16136 16137 16138 16139 16140 16141 16142 16143 16144 16145 16146 16147 16148 16149 16150 16151 16152 16153 16154 16155 16156 16157 16158 16159 16160 16161 16162 16163 16164 16165 16166 16167 16168 16169 16170 16171 16172 16173 16174 16175 16176 16177 16178 16179 16180 16181 16182 16183 16184 16185 16186 16187 16188 16189 16190 16191 16192 16193 16194 16195 16196 16197 16198 16199 16200 16201 16202 16203 16204 16205 16206 16207 16208 16209 16210 16211 16212 16213 16214 16215 16216 16217 16218 16219 16220 16221 16222 16223 16224 16225 16226 16227 16228 16229 16230 16231 16232 16233 16234 16235 16236 16237 16238 16239 16240 16241 16242 16243 16244 16245 16246 16247 16248 16249 16250 16251 16252 16253 16254 16255 16256 16257 16258 16259 16260 16261 16262 16263 16264 16265 16266 16267 16268 16269 16270 16271 16272 16273 16274 16275 16276 16277 16278 16279 16280 16281 16282 16283 16284 16285 16286 16287 16288 16289 16290 16291 16292 16293 16294 16295 16296 16297 16298 16299 16300 16301 16302 16303 16304 16305 16306 16307 16308 16309 16310 16311 16312 16313 16314 16315 16316 16317 16318 16319 16320 16321 16322 16323 16324 16325 16326 16327 16328 16329 16330 16331 16332 16333 16334 16335 16336 16337 16338 16339 16340 16341 16342 16343 16344 16345 16346 16347 16348 16349 16350 16351 16352 16353 16354 16355 16356 16357 16358 16359 16360 16361 16362 16363 16364 16365 16366 16367 16368 16369 16370 16371 16372 16373 16374 16375 16376 16377 16378 16379 16380 16381 16382 16383 16384 16385 16386 16387 16388 16389 16390 16391 16392 16393 16394 16395 16396 16397 16398 16399 16400 16401 16402 16403 16404 16405 16406 16407 16408 16409 16410 16411 16412 16413 16414 16415 16416 16417 16418 16419 16420 16421 16422 16423 16424 16425 16426 16427 16428 16429 16430 16431 16432 16433 16434 16435 16436 16437 16438 16439 16440 16441 16442 16443 16444 16445 16446 16447 16448 16449 16450 16451 16452 16453 16454 16455 16456 16457 16458 16459 16460 16461 16462 16463 16464 16465 16466 16467 16468 16469 16470 16471 16472 16473 16474 16475 16476 16477 16478 16479 16480 16481 16482 16483 16484 16485 16486 16487 16488 16489 16490 16491 16492 16493 16494 16495 16496 16497 16498 16499 16500 16501 16502 16503 16504 16505 16506 16507 16508 16509 16510 16511 16512 16513 16514 16515 16516 16517 16518 16519 16520 16521 16522 16523 16524 16525 16526 16527 16528 16529 16530 16531 16532 16533 16534 16535 16536 16537 16538 16539 16540 16541 16542 16543 16544 16545 16546 16547 16548 16549 16550 16551 16552 16553 16554 16555 16556 16557 16558 16559 16560 16561 16562 16563 16564 16565 16566 16567 16568 16569 16570 16571 16572 16573 16574 16575 16576 16577 16578 16579 16580 16581 16582 16583 16584 16585 16586 16587 16588 16589 16590 16591 16592 16593 16594 16595 16596 16597 16598 16599 16600 16601 16602 16603 16604 16605 16606 16607 16608 16609 16610 16611 16612 16613 16614 16615 16616 16617 16618 16619 16620 16621 16622 16623 16624 16625 16626 16627 16628 16629 16630 16631 16632 16633 16634 16635 16636 16637 16638 16639 16640 16641 16642 16643 16644 16645 16646 16647 16648 16649 16650 16651 16652 16653 16654 16655 16656 16657 16658 16659 16660 16661 16662 16663 16664 16665 16666 16667 16668 16669 16670 16671 16672 16673 16674 16675 16676 16677 16678 16679 16680 16681 16682 16683 16684 16685 16686 16687 16688 16689 16690 16691 16692 16693 16694 16695 16696 16697 16698 16699 16700 16701 16702 16703 16704 16705 16706 16707 16708 16709 16710 16711 16712 16713 16714 16715 16716 16717 16718 16719 16720 16721 16722 16723 16724 16725 16726 16727 16728 16729 16730 16731 16732 16733 16734 16735 16736 16737 16738 16739 16740 16741 16742 16743 16744 16745 16746 16747 16748 16749 16750 16751 16752 16753 16754 16755 16756 16757 16758 16759 16760 16761 16762 16763 16764 16765 16766 16767 16768 16769 16770 16771 16772 16773 16774 16775 16776 16777 16778 16779 16780 16781 16782 16783 16784 16785 16786 16787 16788 16789 16790 16791 16792 16793 16794 16795 16796 16797 16798 16799 16800 16801 16802 16803 16804 16805 16806 16807 16808 16809 16810 16811 16812 16813 16814 16815 16816 16817 16818 16819 16820 16821 16822 16823 16824 16825 16826 16827 16828 16829 16830 16831 16832 16833 16834 16835 16836 16837 16838 16839 16840 16841 16842 16843 16844 16845 16846 16847 16848 16849 16850 16851 16852 16853 16854 16855 16856 16857 16858 16859 16860 16861 16862 16863 16864 16865 16866 16867 16868 16869 16870 16871 16872 16873 16874 16875 16876 16877 16878 16879 16880 16881 16882 16883 16884 16885 16886 16887 16888 16889 16890 16891 16892 16893 16894 16895 16896 16897 16898 16899 16900 16901 16902 16903 16904 16905 16906 16907 16908 16909 16910 16911 16912 16913 16914 16915 16916 16917 16918 16919 16920 16921 16922 16923 16924 16925 16926 16927 16928 16929 16930 16931 16932 16933 16934 16935 16936 16937 16938 16939 16940 16941 16942 16943 16944 16945 16946 16947 16948 16949 16950 16951 16952 16953 16954 16955 16956 16957 16958 16959 16960 16961 16962 16963 16964 16965 16966 16967 16968 16969 16970 16971 16972 16973 16974 16975 16976 16977 16978 16979 16980 16981 16982 16983 16984 16985 16986 16987 16988 16989 16990 16991 16992 16993 16994 16995 16996 16997 16998 16999 17000 17001 17002 17003 17004 17005 17006 17007 17008 17009 17010 17011 17012 17013 17014 17015 17016 17017 17018 17019 17020 17021 17022 17023 17024 17025 17026 17027 17028 17029 17030 17031 17032 17033 17034 17035 17036 17037 17038 17039 17040 17041 17042 17043 17044 17045 17046 17047 17048 17049 17050 17051 17052 17053 17054 17055 17056 17057 17058 17059 17060 17061 17062 17063 17064 17065 17066 17067 17068 17069 17070 17071 17072 17073 17074 17075 17076 17077 17078 17079 17080 17081 17082 17083 17084 17085 17086 17087 17088 17089 17090 17091 17092 17093 17094 17095 17096 17097 17098 17099 17100 17101 17102 17103 17104 17105 17106 17107 17108 17109 17110 17111 17112 17113 17114 17115 17116 17117 17118 17119 17120 17121 17122 17123 17124 17125 17126 17127 17128 17129 17130 17131 17132 17133 17134 17135 17136 17137 17138 17139 17140 17141 17142 17143 17144 17145 17146 17147 17148 17149 17150 17151 17152 17153 17154 17155 17156 17157 17158 17159 17160 17161 17162 17163 17164 17165 17166 17167 17168 17169 17170 17171 17172 17173 17174 17175 17176 17177 17178 17179 17180 17181 17182 17183 17184 17185 17186 17187 17188 17189 17190 17191 17192 17193 17194 17195 17196 17197 17198 17199 17200 17201 17202 17203 17204 17205 17206 17207 17208 17209 17210 17211 17212 17213 17214 17215 17216 17217 17218 17219 17220 17221 17222 17223 17224 17225 17226 17227 17228 17229 17230 17231 17232 17233 17234 17235 17236 17237 17238 17239 17240 17241 17242 17243 17244 17245 17246 17247 17248 17249 17250 17251 17252 17253 17254 17255 17256 17257 17258 17259 17260 17261 17262 17263 17264 17265 17266 17267 17268 17269 17270 17271 17272 17273 17274 17275 17276 17277 17278 17279 17280 17281 17282 17283 17284 17285 17286 17287 17288 17289 17290 17291 17292 17293 17294 17295 17296 17297 17298 17299 17300 17301 17302 17303 17304 17305 17306 17307 17308 17309 17310 17311 17312 17313 17314 17315 17316 17317 17318 17319 17320 17321 17322 17323 17324 17325 17326 17327 17328 17329 17330 17331 17332 17333 17334 17335 17336 17337 17338 17339 17340 17341 17342 17343 17344 17345 17346 17347 17348 17349 17350 17351 17352 17353 17354 17355 17356 17357 17358 17359 17360 17361 17362 17363 17364 17365 17366 17367 17368 17369 17370 17371 17372 17373 17374 17375 17376 17377 17378 17379 17380 17381 17382 17383 17384 17385 17386 17387 17388 17389 17390 17391 17392 17393 17394 17395 17396 17397 17398 17399 17400 17401 17402 17403 17404 17405 17406 17407 17408 17409 17410 17411 17412 17413 17414 17415 17416 17417 17418 17419 17420 17421 17422 17423 17424 17425 17426 17427 17428 17429 17430 17431 17432 17433 17434 17435 17436 17437 17438 17439 17440 17441 17442 17443 17444 17445 17446 17447 17448 17449 17450 17451 17452 17453 17454 17455 17456 17457 17458 17459 17460 17461 17462 17463 17464 17465 17466 17467 17468 17469 17470 17471 17472 17473 17474 17475 17476 17477 17478 17479 17480 17481 17482 17483 17484 17485 17486 17487 17488 17489 17490 17491 17492 17493 17494 17495 17496 17497 17498 17499 17500 17501 17502 17503 17504 17505 17506 17507 17508 17509 17510 17511 17512 17513 17514 17515 17516 17517 17518 17519 17520 17521 17522 17523 17524 17525 17526 17527 17528 17529 17530 17531 17532 17533 17534 17535 17536 17537 17538 17539 17540 17541 17542 17543 17544 17545 17546 17547 17548 17549 17550 17551 17552 17553 17554 17555 17556 17557 17558 17559 17560 17561 17562 17563 17564 17565 17566 17567 17568 17569 17570 17571 17572 17573 17574 17575 17576 17577 17578 17579 17580 17581 17582 17583 17584 17585 17586 17587 17588 17589 17590 17591 17592 17593 17594 17595 17596 17597 17598 17599 17600 17601 17602 17603 17604 17605 17606 17607 17608 17609 17610 17611 17612 17613 17614 17615 17616 17617 17618 17619 17620 17621 17622 17623 17624 17625 17626 17627 17628 17629 17630 17631 17632 17633 17634 17635 17636 17637 17638 17639 17640 17641 17642 17643 17644 17645 17646 17647 17648 17649 17650 17651 17652 17653 17654 17655 17656 17657 17658 17659 17660 17661 17662 17663 17664 17665 17666 17667 17668 17669 17670 17671 17672 17673 17674 17675 17676 17677 17678 17679 17680 17681 17682 17683 17684 17685 17686 17687 17688 17689 17690 17691 17692 17693 17694 17695 17696 17697 17698 17699 17700 17701 17702 17703 17704 17705 17706 17707 17708 17709 17710 17711 17712 17713 17714 17715 17716 17717 17718 17719 17720 17721 17722 17723 17724 17725 17726 17727 17728 17729 17730 17731 17732 17733 17734 17735 17736 17737 17738 17739 17740 17741 17742 17743 17744 17745 17746 17747 17748 17749 17750 17751 17752 17753 17754 17755 17756 17757 17758 17759 17760 17761 17762 17763 17764 17765 17766 17767 17768 17769 17770 17771 17772 17773 17774 17775 17776 17777 17778 17779 17780 17781 17782 17783 17784 17785 17786 17787 17788 17789 17790 17791 17792 17793 17794 17795 17796 17797 17798 17799 17800 17801 17802 17803 17804 17805 17806 17807 17808 17809 17810 17811 17812 17813 17814 17815 17816 17817 17818 17819 17820 17821 17822 17823 17824 17825 17826 17827 17828 17829 17830 17831 17832 17833 17834 17835 17836 17837 17838 17839 17840 17841 17842 17843 17844 17845 17846 17847 17848 17849 17850 17851 17852 17853 17854 17855 17856 17857 17858 17859 17860 17861 17862 17863 17864 17865 17866 17867 17868 17869 17870 17871 17872 17873 17874 17875 17876 17877 17878 17879 17880 17881 17882 17883 17884 17885 17886 17887 17888 17889 17890 17891 17892 17893 17894 17895 17896 17897 17898 17899 17900 17901 17902 17903 17904 17905 17906 17907 17908 17909 17910 17911 17912 17913 17914 17915 17916 17917 17918 17919 17920 17921 17922 17923 17924 17925 17926 17927 17928 17929 17930 17931 17932 17933 17934 17935 17936 17937 17938 17939 17940 17941 17942 17943 17944 17945 17946 17947 17948 17949 17950 17951 17952 17953 17954 17955 17956 17957 17958 17959 17960 17961 17962 17963 17964 17965 17966 17967 17968 17969 17970 17971 17972 17973 17974 17975 17976 17977 17978 17979 17980 17981 17982 17983 17984 17985 17986 17987 17988 17989 17990 17991 17992 17993 17994 17995 17996 17997 17998 17999 18000 18001 18002 18003 18004 18005 18006 18007 18008 18009 18010 18011 18012 18013 18014 18015 18016 18017 18018 18019 18020 18021 18022 18023 18024 18025 18026 18027 18028 18029 18030 18031 18032 18033 18034 18035 18036 18037 18038 18039 18040 18041 18042 18043 18044 18045 18046 18047 18048 18049 18050 18051 18052 18053 18054 18055 18056 18057 18058 18059 18060 18061 18062 18063 18064 18065 18066 18067 18068 18069 18070 18071 18072 18073 18074 18075 18076 18077 18078 18079 18080 18081 18082 18083 18084 18085 18086 18087 18088 18089 18090 18091 18092 18093 18094 18095 18096 18097 18098 18099 18100 18101 18102 18103 18104 18105 18106 18107 18108 18109 18110 18111 18112 18113 18114 18115 18116 18117 18118 18119 18120 18121 18122 18123 18124 18125 18126 18127 18128 18129 18130 18131 18132 18133 18134 18135 18136 18137 18138 18139 18140 18141 18142 18143 18144 18145 18146 18147 18148 18149 18150 18151 18152 18153 18154 18155 18156 18157 18158 18159 18160 18161 18162 18163 18164 18165 18166 18167 18168 18169 18170 18171 18172 18173 18174 18175 18176 18177 18178 18179 18180 18181 18182 18183 18184 18185 18186 18187 18188 18189 18190 18191 18192 18193 18194 18195 18196 18197 18198 18199 18200 18201 18202 18203 18204 18205 18206 18207 18208 18209 18210 18211 18212 18213 18214 18215 18216 18217 18218 18219 18220 18221 18222 18223 18224 18225 18226 18227 18228 18229 18230 18231 18232 18233 18234 18235 18236 18237 18238 18239 18240 18241 18242 18243 18244 18245 18246 18247 18248 18249 18250 18251 18252 18253 18254 18255 18256 18257 18258 18259 18260 18261 18262 18263 18264 18265 18266 18267 18268 18269 18270 18271 18272 18273 18274 18275 18276 18277 18278 18279 18280 18281 18282 18283 18284 18285 18286 18287 18288 18289 18290 18291 18292 18293 18294 18295 18296 18297 18298 18299 18300 18301 18302 18303 18304 18305 18306 18307 18308 18309 18310 18311 18312 18313 18314 18315 18316 18317 18318 18319 18320 18321 18322 18323 18324 18325 18326 18327 18328 18329 18330 18331 18332 18333 18334 18335 18336 18337 18338 18339 18340 18341 18342 18343 18344 18345 18346 18347 18348 18349 18350 18351 18352 18353 18354 18355 18356 18357 18358 18359 18360 18361 18362 18363 18364 18365 18366 18367 18368 18369 18370 18371 18372 18373 18374 18375 18376 18377 18378 18379 18380 18381 18382 18383 18384 18385 18386 18387 18388 18389 18390 18391 18392 18393 18394 18395 18396 18397 18398 18399 18400 18401 18402 18403 18404 18405 18406 18407 18408 18409 18410 18411 18412 18413 18414 18415 18416 18417 18418 18419 18420 18421 18422 18423 18424 18425 18426 18427 18428 18429 18430 18431 18432 18433 18434 18435 18436 18437 18438 18439 18440 18441 18442 18443 18444 18445 18446 18447 18448 18449 18450 18451 18452 18453 18454 18455 18456 18457 18458 18459 18460 18461 18462 18463 18464 18465 18466 18467 18468 18469 18470 18471 18472 18473 18474 18475 18476 18477 18478 18479 18480 18481 18482 18483 18484 18485 18486 18487 18488 18489 18490 18491 18492 18493 18494 18495 18496 18497 18498 18499 18500 18501 18502 18503 18504 18505 18506 18507 18508 18509 18510 18511 18512 18513 18514 18515 18516 18517 18518 18519 18520 18521 18522 18523 18524 18525 18526 18527 18528 18529 18530 18531 18532 18533 18534 18535 18536 18537 18538 18539 18540 18541 18542 18543 18544 18545 18546 18547 18548 18549 18550 18551 18552 18553 18554 18555 18556 18557 18558 18559 18560 18561 18562 18563 18564 18565 18566 18567 18568 18569 18570 18571 18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 18595 18596 18597 18598 18599 18600 18601 18602 18603 18604 18605 18606 18607 18608 18609 18610 18611 18612 18613 18614 18615 18616 18617 18618 18619 18620 18621 18622 18623 18624 18625 18626 18627 18628 18629 18630 18631 18632 18633 18634 18635 18636 18637 18638 18639 18640 18641 18642 18643 18644 18645 18646 18647 18648 18649 18650 18651 18652 18653 18654 18655 18656 18657 18658 18659 18660 18661 18662 18663 18664 18665 18666 18667 18668 18669 18670 18671 18672 18673 18674 18675 18676 18677 18678 18679 18680 18681 18682 18683 18684 18685 18686 18687 18688 18689 18690 18691 18692 18693 18694 18695 18696 18697 18698 18699 18700 18701 18702 18703 18704 18705 18706 18707 18708 18709 18710 18711 18712 18713 18714 18715 18716 18717 18718 18719 18720 18721 18722 18723 18724 18725 18726 18727 18728 18729 18730 18731 18732 18733 18734 18735 18736 18737 18738 18739 18740 18741 18742 18743 18744 18745 18746 18747 18748 18749 18750 18751 18752 18753 18754 18755 18756 18757 18758 18759 18760 18761 18762 18763 18764 18765 18766 18767 18768 18769 18770 18771 18772 18773 18774 18775 18776 18777 18778 18779 18780 18781 18782 18783 18784 18785 18786 18787 18788 18789 18790 18791 18792 18793 18794 18795 18796 18797 18798 18799 18800 18801 18802 18803 18804 18805 18806 18807 18808 18809 18810 18811 18812 18813 18814 18815 18816 18817 18818 18819 18820 18821 18822 18823 18824 18825 18826 18827 18828 18829 18830 18831 18832 18833 18834 18835 18836 18837 18838 18839 18840 18841 18842 18843 18844 18845 18846 18847 18848 18849 18850 18851 18852 18853 18854 18855 18856 18857 18858 18859 18860 18861 18862 18863 18864 18865 18866 18867 18868 18869 18870 18871 18872 18873 18874 18875 18876 18877 18878 18879 18880 18881 18882 18883 18884 18885 18886 18887 18888 18889 18890 18891 18892 18893 18894 18895 18896 18897 18898 18899 18900 18901 18902 18903 18904 18905 18906 18907 18908 18909 18910 18911 18912 18913 18914 18915 18916 18917 18918 18919 18920 18921 18922 18923 18924 18925 18926 18927 18928 18929 18930 18931 18932 18933 18934 18935 18936 18937 18938 18939 18940 18941 18942 18943 18944 18945 18946 18947 18948 18949 18950 18951 18952 18953 18954 18955 18956 18957 18958 18959 18960 18961 18962 18963 18964 18965 18966 18967 18968 18969 18970 18971 18972 18973 18974 18975 18976 18977 18978 18979 18980 18981 18982 18983 18984 18985 18986 18987 18988 18989 18990 18991 18992 18993 18994 18995 18996 18997 18998 18999 19000 19001 19002 19003 19004 19005 19006 19007 19008 19009 19010 19011 19012 19013 19014 19015 19016 19017 19018 19019 19020 19021 19022 19023 19024 19025 19026 19027 19028 19029 19030 19031 19032 19033 19034 19035 19036 19037 19038 19039 19040 19041 19042 19043 19044 19045 19046 19047 19048 19049 19050 19051 19052 19053 19054 19055 19056 19057 19058 19059 19060 19061 19062 19063 19064 19065 19066 19067 19068 19069 19070 19071 19072 19073 19074 19075 19076 19077 19078 19079 19080 19081 19082 19083 19084 19085 19086 19087 19088 19089 19090 19091 19092 19093 19094 19095 19096 19097 19098 19099 19100 19101 19102 19103 19104 19105 19106 19107 19108 19109 19110 19111 19112 19113 19114 19115 19116 19117 19118 19119 19120 19121 19122 19123 19124 19125 19126 19127 19128 19129 19130 19131 19132 19133 19134 19135 19136 19137 19138 19139 19140 19141 19142 19143 19144 19145 19146 19147 19148 19149 19150 19151 19152 19153 19154 19155 19156 19157 19158 19159 19160 19161 19162 19163 19164 19165 19166 19167 19168 19169 19170 19171 19172 19173 19174 19175 19176 19177 19178 19179 19180 19181 19182 19183 19184 19185 19186 19187 19188 19189 19190 19191 19192 19193 19194 19195 19196 19197 19198 19199 19200 19201 19202 19203 19204 19205 19206 19207 19208 19209 19210 19211 19212 19213 19214 19215 19216 19217 19218 19219 19220 19221 19222 19223 19224 19225 19226 19227 19228 19229 19230 19231 19232 19233 19234 19235 19236 19237 19238 19239 19240 19241 19242 19243 19244 19245 19246 19247 19248 19249 19250 19251 19252 19253 19254 19255 19256 19257 19258 19259 19260 19261 19262 19263 19264 19265 19266 19267 19268 19269 19270 19271 19272 19273 19274 19275 19276 19277 19278 19279 19280 19281 19282 19283 19284 19285 19286 19287 19288 19289 19290 19291 19292 19293 19294 19295 19296 19297 19298 19299 19300 19301 19302 19303 19304 19305 19306 19307 19308 19309 19310 19311 19312 19313 19314 19315 19316 19317 19318 19319 19320 19321 19322 19323 19324 19325 19326 19327 19328 19329 19330 19331 19332 19333 19334 19335 19336 19337 19338 19339 19340 19341 19342 19343 19344 19345 19346 19347 19348 19349 19350 19351 19352 19353 19354 19355 19356 19357 19358 19359 19360 19361 19362 19363 19364 19365 19366 19367 19368 19369 19370 19371 19372 19373 19374 19375 19376 19377 19378 19379 19380 19381 19382 19383 19384 19385 19386 19387 19388 19389 19390 19391 19392 19393 19394 19395 19396 19397 19398 19399 19400 19401 19402 19403 19404 19405 19406 19407 19408 19409 19410 19411 19412 19413 19414 19415 19416 19417 19418 19419 19420 19421 19422 19423 19424 19425 19426 19427 19428 19429 19430 19431 19432 19433 19434 19435 19436 19437 19438 19439 19440 19441 19442 19443 19444 19445 19446 19447 19448 19449 19450 19451 19452 19453 19454 19455 19456 19457 19458 19459 19460 19461 19462 19463 19464 19465 19466 19467 19468 19469 19470 19471 19472 19473 19474 19475 19476 19477 19478 19479 19480 19481 19482 19483 19484 19485 19486 19487 19488 19489 19490 19491 19492 19493 19494 19495 19496 19497 19498 19499 19500 19501 19502 19503 19504 19505 19506 19507 19508 19509 19510 19511 19512 19513 19514 19515 19516 19517 19518 19519 19520 19521 19522 19523 19524 19525 19526 19527 19528 19529 19530 19531 19532 19533 19534 19535 19536 19537 19538 19539 19540 19541 19542 19543 19544 19545 19546 19547 19548 19549 19550 19551 19552 19553 19554 19555 19556 19557 19558 19559 19560 19561 19562 19563 19564 19565 19566 19567 19568 19569 19570 19571 19572 19573 19574 19575 19576 19577 19578 19579 19580 19581 19582 19583 19584 19585 19586 19587 19588 19589 19590 19591 19592 19593 19594 19595 19596 19597 19598 19599 19600 19601 19602 19603 19604 19605 19606 19607 19608 19609 19610 19611 19612 19613 19614 19615 19616 19617 19618 19619 19620 19621 19622 19623 19624 19625 19626 19627 19628 19629 19630 19631 19632 19633 19634 19635 19636 19637 19638 19639 19640 19641 19642 19643 19644 19645 19646 19647 19648 19649 19650 19651 19652 19653 19654 19655 19656 19657 19658 19659 19660 19661 19662 19663 19664 19665 19666 19667 19668 19669 19670 19671 19672 19673 19674 19675 19676 19677 19678 19679 19680 19681 19682 19683 19684 19685 19686 19687 19688 19689 19690 19691 19692 19693 19694 19695 19696 19697 19698 19699 19700 19701 19702 19703 19704 19705 19706 19707 19708 19709 19710 19711 19712 19713 19714 19715 19716 19717 19718 19719 19720 19721 19722 19723 19724 19725 19726 19727 19728 19729 19730 19731 19732 19733 19734 19735 19736 19737 19738 19739 19740 19741 19742 19743 19744 19745 19746 19747 19748 19749 19750 19751 19752 19753 19754 19755 19756 19757 19758 19759 19760 19761 19762 19763 19764 19765 19766 19767 19768 19769 19770 19771 19772 19773 19774 19775 19776 19777 19778 19779 19780 19781 19782 19783 19784 19785 19786 19787 19788 19789 19790 19791 19792 19793 19794 19795 19796 19797 19798 19799 19800 19801 19802 19803 19804 19805 19806 19807 19808 19809 19810 19811 19812 19813 19814 19815 19816 19817 19818 19819 19820 19821 19822 19823 19824 19825 19826 19827 19828 19829 19830 19831 19832 19833 19834 19835 19836 19837 19838 19839 19840 19841 19842 19843 19844 19845 19846 19847 19848 19849 19850 19851 19852 19853 19854 19855 19856 19857 19858 19859 19860 19861 19862 19863 19864 19865 19866 19867 19868 19869 19870 19871 19872 19873 19874 19875 19876 19877 19878 19879 19880 19881 19882 19883 19884 19885 19886 19887 19888 19889 19890 19891 19892 19893 19894 19895 19896 19897 19898 19899 19900 19901 19902 19903 19904 19905 19906 19907 19908 19909 19910 19911 19912 19913 19914 19915 19916 19917 19918 19919 19920 19921 19922 19923 19924 19925 19926 19927 19928 19929 19930 19931 19932 19933 19934 19935 19936 19937 19938 19939 19940 19941 19942 19943 19944 19945 19946 19947 19948 19949 19950 19951 19952 19953 19954 19955 19956 19957 19958 19959 19960 19961 19962 19963 19964 19965 19966 19967 19968 19969 19970 19971 19972 19973 19974 19975 19976 19977 19978 19979 19980 19981 19982 19983 19984 19985 19986 19987 19988 19989 19990 19991 19992 19993 19994 19995 19996 19997 19998 19999 20000 20001 20002 20003 20004 20005 20006 20007 20008 20009 20010 20011 20012 20013 20014 20015 20016 20017 20018 20019 20020 20021 20022 20023 20024 20025 20026 20027 20028 20029 20030 20031 20032 20033 20034 20035 20036 20037 20038 20039 20040 20041 20042 20043 20044 20045 20046 20047 20048 20049 20050 20051 20052 20053 20054 20055 20056 20057 20058 20059 20060 20061 20062 20063 20064 20065 20066 20067 20068 20069 20070 20071 20072 20073 20074 20075 20076 20077 20078 20079 20080 20081 20082 20083 20084 20085 20086 20087 20088 20089 20090 20091 20092 20093 20094 20095 20096 20097 20098 20099 20100 20101 20102 20103 20104 20105 20106 20107 20108 20109 20110 20111 20112 20113 20114 20115 20116 20117 20118 20119 20120 20121 20122 20123 20124 20125 20126 20127 20128 20129 20130 20131 20132 20133 20134 20135 20136 20137 20138 20139 20140 20141 20142 20143 20144 20145 20146 20147 20148 20149 20150 20151 20152 20153 20154 20155 20156 20157 20158 20159 20160 20161 20162 20163 20164 20165 20166 20167 20168 20169 20170 20171 20172 20173 20174 20175 20176 20177 20178 20179 20180 20181 20182 20183 20184 20185 20186 20187 20188 20189 20190 20191 20192 20193 20194 20195 20196 20197 20198 20199 20200 20201 20202 20203 20204 20205 20206 20207 20208 20209 20210 20211 20212 20213 20214 20215 20216 20217 20218 20219 20220 20221 20222 20223 20224 20225 20226 20227 20228 20229 20230 20231 20232 20233 20234 20235 20236 20237 20238 20239 20240 20241 20242 20243 20244 20245 20246 20247 20248 20249 20250 20251 20252 20253 20254 20255 20256 20257 20258 20259 20260 20261 20262 20263 20264 20265 20266 20267 20268 20269 20270 20271 20272 20273 20274 20275 20276 20277 20278 20279 20280 20281 20282 20283 20284 20285 20286 20287 20288 20289 20290 20291 20292 20293 20294 20295 20296 20297 20298 20299 20300 20301 20302 20303 20304 20305 20306 20307 20308 20309 20310 20311 20312 20313 20314 20315 20316 20317 20318 20319 20320 20321 20322 20323 20324 20325 20326 20327 20328 20329 20330 20331 20332 20333 20334 20335 20336 20337 20338 20339 20340 20341 20342 20343 20344 20345 20346 20347 20348 20349 20350 20351 20352 20353 20354 20355 20356 20357 20358 20359 20360 20361 20362 20363 20364 20365 20366 20367 20368 20369 20370 20371 20372 20373 20374 20375 20376 20377 20378 20379 20380 20381 20382 20383 20384 20385 20386 20387 20388 20389 20390 20391 20392 20393 20394 20395 20396 20397 20398 20399 20400 20401 20402 20403 20404 20405 20406 20407 20408 20409 20410 20411 20412 20413 20414 20415 20416 20417 20418 20419 20420 20421 20422 20423 20424 20425 20426 20427 20428 20429 20430 20431 20432 20433 20434 20435 20436 20437 20438 20439 20440 20441 20442 20443 20444 20445 20446 20447 20448 20449 20450 20451 20452 20453 20454 20455 20456 20457 20458 20459 20460 20461 20462 20463 20464 20465 20466 20467 20468 20469 20470 20471 20472 20473 20474 20475 20476 20477 20478 20479 20480 20481 20482 20483 20484 20485 20486 20487 20488 20489 20490 20491 20492 20493 20494 20495 20496 20497 20498 20499 20500 20501 20502 20503 20504 20505 20506 20507 20508 20509 20510 20511 20512 20513 20514 20515 20516 20517 20518 20519 20520 20521 20522 20523 20524 20525 20526 20527 20528 20529 20530 20531 20532 20533 20534 20535 20536 20537 20538 20539 20540 20541 20542 20543 20544 20545 20546 20547 20548 20549 20550 20551 20552 20553 20554 20555 20556 20557 20558 20559 20560 20561 20562 20563 20564 20565 20566 20567 20568 20569 20570 20571 20572 20573 20574 20575 20576 20577 20578 20579 20580 20581 20582 20583 20584 20585 20586 20587 20588 20589 20590 20591 20592 20593 20594 20595 20596 20597 20598 20599 20600 20601 20602 20603 20604 20605 20606 20607 20608 20609 20610 20611 20612 20613 20614 20615 20616 20617 20618 20619 20620 20621 20622 20623 20624 20625 20626 20627 20628 20629 20630 20631 20632 20633 20634 20635 20636 20637 20638 20639 20640 20641 20642 20643 20644 20645 20646 20647 20648 20649 20650 20651 20652 20653 20654 20655 20656 20657 20658 20659 20660 20661 20662 20663 20664 20665 20666 20667 20668 20669 20670 20671 20672 20673 20674 20675 20676 20677 20678 20679 20680 20681 20682 20683 20684 20685 20686 20687 20688 20689 20690 20691 20692 20693 20694 20695 20696 20697 20698 20699 20700 20701 20702 20703 20704 20705 20706 20707 20708 20709 20710 20711 20712 20713 20714 20715 20716 20717 20718 20719 20720 20721 20722 20723 20724 20725 20726 20727 20728 20729 20730 20731 20732 20733 20734 20735 20736 20737 20738 20739 20740 20741 20742 20743 20744 20745 20746 20747 20748 20749 20750 20751 20752 20753 20754 20755 20756 20757 20758 20759 20760 20761 20762 20763 20764 20765 20766 20767 20768 20769 20770 20771 20772 20773 20774 20775 20776 20777 20778 20779 20780 20781 20782 20783 20784 20785 20786 20787 20788 20789 20790 20791 20792 20793 20794 20795 20796 20797 20798 20799 20800 20801 20802 20803 20804 20805 20806 20807 20808 20809 20810 20811 20812 20813 20814 20815 20816 20817 20818 20819 20820 20821 20822 20823 20824 20825 20826 20827 20828 20829 20830 20831 20832 20833 20834 20835 20836 20837 20838 20839 20840 20841 20842 20843 20844 20845 20846 20847 20848 20849 20850 20851 20852 20853 20854 20855 20856 20857 20858 20859 20860 20861 20862 20863 20864 20865 20866 20867 20868 20869 20870 20871 20872 20873 20874 20875 20876 20877 20878 20879 20880 20881 20882 20883 20884 20885 20886 20887 20888 20889 20890 20891 20892 20893 20894 20895 20896 20897 20898 20899 20900 20901 20902 20903 20904 20905 20906 20907 20908 20909 20910 20911 20912 20913 20914 20915 20916 20917 20918 20919 20920 20921 20922 20923 20924 20925 20926 20927 20928 20929 20930 20931 20932 20933 20934 20935 20936 20937 20938 20939 20940 20941 20942 20943 20944 20945 20946 20947 20948 20949 20950 20951 20952 20953 20954 20955 20956 20957 20958 20959 20960 20961 20962 20963 20964 20965 20966 20967 20968 20969 20970 20971 20972 20973 20974 20975 20976 20977 20978 20979 20980 20981 20982 20983 20984 20985 20986 20987 20988 20989 20990 20991 20992 20993 20994 20995 20996 20997 20998 20999 21000 21001 21002 21003 21004 21005 21006 21007 21008 21009 21010 21011 21012 21013 21014 21015 21016 21017 21018 21019 21020 21021 21022 21023 21024 21025 21026 21027 21028 21029 21030 21031 21032 21033 21034 21035 21036 21037 21038 21039 21040 21041 21042 21043 21044 21045 21046 21047 21048 21049 21050 21051 21052 21053 21054 21055 21056 21057 21058 21059 21060 21061 21062 21063 21064 21065 21066 21067 21068 21069 21070 21071 21072 21073 21074 21075 21076 21077 21078 21079 21080 21081 21082 21083 21084 21085 21086 21087 21088 21089 21090 21091 21092 21093 21094 21095 21096 21097 21098 21099 21100 21101 21102 21103 21104 21105 21106 21107 21108 21109 21110 21111 21112 21113 21114 21115 21116 21117 21118 21119 21120 21121 21122 21123 21124 21125 21126 21127 21128 21129 21130 21131 21132 21133 21134 21135 21136 21137 21138 21139 21140 21141 21142 21143 21144 21145 21146 21147 21148 21149 21150 21151 21152 21153 21154 21155 21156 21157 21158 21159 21160 21161 21162 21163 21164 21165 21166 21167 21168 21169 21170 21171 21172 21173 21174 21175 21176 21177 21178 21179 21180 21181 21182 21183 21184 21185 21186 21187 21188 21189 21190 21191 21192 21193 21194 21195 21196 21197 21198 21199 21200 21201 21202 21203 21204 21205 21206 21207 21208 21209 21210 21211 21212 21213 21214 21215 21216 21217 21218 21219 21220 21221 21222 21223 21224 21225 21226 21227 21228 21229 21230 21231 21232 21233 21234 21235 21236 21237 21238 21239 21240 21241 21242 21243 21244 21245 21246 21247 21248 21249 21250 21251 21252 21253 21254 21255 21256 21257 21258 21259 21260 21261 21262 21263 21264 21265 21266 21267 21268 21269 21270 21271 21272 21273 21274 21275 21276 21277 21278 21279 21280 21281 21282 21283 21284 21285 21286 21287 21288 21289 21290 21291 21292 21293 21294 21295 21296 21297 21298 21299 21300 21301 21302 21303 21304 21305 21306 21307 21308 21309 21310 21311 21312 21313 21314 21315 21316 21317 21318 21319 21320 21321 21322 21323 21324 21325 21326 21327 21328 21329 21330 21331 21332 21333 21334 21335 21336 21337 21338 21339 21340 21341 21342 21343 21344 21345 21346 21347 21348 21349 21350 21351 21352 21353 21354 21355 21356 21357 21358 21359 21360 21361 21362 21363 21364 21365 21366 21367 21368 21369 21370 21371 21372 21373 21374 21375 21376 21377 21378 21379 21380 21381 21382 21383 21384 21385 21386 21387 21388 21389 21390 21391 21392 21393 21394 21395 21396 21397 21398 21399 21400 21401 21402 21403 21404 21405 21406 21407 21408 21409 21410 21411 21412 21413 21414 21415 21416 21417 21418 21419 21420 21421 21422 21423 21424 21425 21426 21427 21428 21429 21430 21431 21432 21433 21434 21435 21436 21437 21438 21439 21440 21441 21442 21443 21444 21445 21446 21447 21448 21449 21450 21451 21452 21453 21454 21455 21456 21457 21458 21459 21460 21461 21462 21463 21464 21465 21466 21467 21468 21469 21470 21471 21472 21473 21474 21475 21476 21477 21478 21479 21480 21481 21482 21483 21484 21485 21486 21487 21488 21489 21490 21491 21492 21493 21494 21495 21496 21497 21498 21499 21500 21501 21502 21503 21504 21505 21506 21507 21508 21509 21510 21511 21512 21513 21514 21515 21516 21517 21518 21519 21520 21521 21522 21523 21524 21525 21526 21527 21528 21529 21530 21531 21532 21533 21534 21535 21536 21537 21538 21539 21540 21541 21542 21543 21544 21545 21546 21547 21548 21549 21550 21551 21552 21553 21554 21555 21556 21557 21558 21559 21560 21561 21562 21563 21564 21565 21566 21567 21568 21569 21570 21571 21572 21573 21574 21575 21576 21577 21578 21579 21580 21581 21582 21583 21584 21585 21586 21587 21588 21589 21590 21591 21592 21593 21594 21595 21596 21597 21598 21599 21600 21601 21602 21603 21604 21605 21606 21607 21608 21609 21610 21611 21612 21613 21614 21615 21616 21617 21618 21619 21620 21621 21622 21623 21624 21625 21626 21627 21628 21629 21630 21631 21632 21633 21634 21635 21636 21637 21638 21639 21640 21641 21642 21643 21644 21645 21646 21647 21648 21649 21650 21651 21652 21653 21654 21655 21656 21657 21658 21659 21660 21661 21662 21663 21664 21665 21666 21667 21668 21669 21670 21671 21672 21673 21674 21675 21676 21677 21678 21679 21680 21681 21682 21683 21684 21685 21686 21687 21688 21689 21690 21691 21692 21693 21694 21695 21696 21697 21698 21699 21700 21701 21702 21703 21704 21705 21706 21707 21708 21709 21710 21711 21712 21713 21714 21715 21716 21717 21718 21719 21720 21721 21722 21723 21724 21725 21726 21727 21728 21729 21730 21731 21732 21733 21734 21735 21736 21737 21738 21739 21740 21741 21742 21743 21744 21745 21746 21747 21748 21749 21750 21751 21752 21753 21754 21755 21756 21757 21758 21759 21760 21761 21762 21763 21764 21765 21766 21767 21768 21769 21770 21771 21772 21773 21774 21775 21776 21777 21778 21779 21780 21781 21782 21783 21784 21785 21786 21787 21788 21789 21790 21791 21792 21793 21794 21795 21796 21797 21798 21799 21800 21801 21802 21803 21804 21805 21806 21807 21808 21809 21810 21811 21812 21813 21814 21815 21816 21817 21818 21819 21820 21821 21822 21823 21824 21825 21826 21827 21828 21829 21830 21831 21832 21833 21834 21835 21836 21837 21838 21839 21840 21841 21842 21843 21844 21845 21846 21847 21848 21849 21850 21851 21852 21853 21854 21855 21856 21857 21858 21859 21860 21861 21862 21863 21864 21865 21866 21867 21868 21869 21870 21871 21872 21873 21874 21875 21876 21877 21878 21879 21880 21881 21882 21883 21884 21885 21886 21887 21888 21889 21890 21891 21892 21893 21894 21895 21896 21897 21898 21899 21900 21901 21902 21903 21904 21905 21906 21907 21908 21909 21910 21911 21912 21913 21914 21915 21916 21917 21918 21919 21920 21921 21922 21923 21924 21925 21926 21927 21928 21929 21930 21931 21932 21933 21934 21935 21936 21937 21938 21939 21940 21941 21942 21943 21944 21945 21946 21947 21948 21949 21950 21951 21952 21953 21954 21955 21956 21957 21958 21959 21960 21961 21962 21963 21964 21965 21966 21967 21968 21969 21970 21971 21972 21973 21974 21975 21976 21977 21978 21979 21980 21981 21982 21983 21984 21985 21986 21987 21988 21989 21990 21991 21992 21993 21994 21995 21996 21997 21998 21999 22000 22001 22002 22003 22004 22005 22006 22007 22008 22009 22010 22011 22012 22013 22014 22015 22016 22017 22018 22019 22020 22021 22022 22023 22024 22025 22026 22027 22028 22029 22030 22031 22032 22033 22034 22035 22036 22037 22038 22039 22040 22041 22042 22043 22044 22045 22046 22047 22048 22049 22050 22051 22052 22053 22054 22055 22056 22057 22058 22059 22060 22061 22062 22063 22064 22065 22066 22067 22068 22069 22070 22071 22072 22073 22074 22075 22076 22077 22078 22079 22080 22081 22082 22083 22084 22085 22086 22087 22088 22089 22090 22091 22092 22093 22094 22095 22096 22097 22098 22099 22100 22101 22102 22103 22104 22105 22106 22107 22108 22109 22110 22111 22112 22113 22114 22115 22116 22117 22118 22119 22120 22121 22122 22123 22124 22125 22126 22127 22128 22129 22130 22131 22132 22133 22134 22135 22136 22137 22138 22139 22140 22141 22142 22143 22144 22145 22146 22147 22148 22149 22150 22151 22152 22153 22154 22155 22156 22157 22158 22159 22160 22161 22162 22163 22164 22165 22166 22167 22168 22169 22170 22171 22172 22173 22174 22175 22176 22177 22178 22179 22180 22181 22182 22183 22184 22185 22186 22187 22188 22189 22190 22191 22192 22193 22194 22195 22196 22197 22198 22199 22200 22201 22202 22203 22204 22205 22206 22207 22208 22209 22210 22211 22212 22213 22214 22215 22216 22217 22218 22219 22220 22221 22222 22223 22224 22225 22226 22227 22228 22229 22230 22231 22232 22233 22234 22235 22236 22237 22238 22239 22240 22241 22242 22243 22244 22245 22246 22247 22248 22249 22250 22251 22252 22253 22254 22255 22256 22257 22258 22259 22260 22261 22262 22263 22264 22265 22266 22267 22268 22269 22270 22271 22272 22273 22274 22275 22276 22277 22278 22279 22280 22281 22282 22283 22284 22285 22286 22287 22288 22289 22290 22291 22292 22293 22294 22295 22296 22297 22298 22299 22300 22301 22302 22303 22304 22305 22306 22307 22308 22309 22310 22311 22312 22313 22314 22315 22316 22317 22318 22319 22320 22321 22322 22323 22324 22325 22326 22327 22328 22329 22330 22331 22332 22333 22334 22335 22336 22337 22338 22339 22340 22341 22342 22343 22344 22345 22346 22347 22348 22349 22350 22351 22352 22353 22354 22355 22356 22357 22358 22359 22360 22361 22362 22363 22364 22365 22366 22367 22368 22369 22370 22371 22372 22373 22374 22375 22376 22377 22378 22379 22380 22381 22382 22383 22384 22385 22386 22387 22388 22389 22390 22391 22392 22393 22394 22395 22396 22397 22398 22399 22400 22401 22402 22403 22404 22405 22406 22407 22408 22409 22410 22411 22412 22413 22414 22415 22416 22417 22418 22419 22420 22421 22422 22423 22424 22425 22426 22427 22428 22429 22430 22431 22432 22433 22434 22435 22436 22437 22438 22439 22440 22441 22442 22443 22444 22445 22446 22447 22448 22449 22450 22451 22452 22453 22454 22455 22456 22457 22458 22459 22460 22461 22462 22463 22464 22465 22466 22467 22468 22469 22470 22471 22472 22473 22474 22475 22476 22477 22478 22479 22480 22481 22482 22483 22484 22485 22486 22487 22488 22489 22490 22491 22492 22493 22494 22495 22496 22497 22498 22499 22500 22501 22502 22503 22504 22505 22506 22507 22508 22509 22510 22511 22512 22513 22514 22515 22516 22517 22518 22519 22520 22521 22522 22523 22524 22525 22526 22527 22528 22529 22530 22531 22532 22533 22534 22535 22536 22537 22538 22539 22540 22541 22542 22543 22544 22545 22546 22547 22548 22549 22550 22551 22552 22553 22554 22555 22556 22557 22558 22559 22560 22561 22562 22563 22564 22565 22566 22567 22568 22569 22570 22571 22572 22573 22574 22575 22576 22577 22578 22579 22580 22581 22582 22583 22584 22585 22586 22587 22588 22589 22590 22591 22592 22593 22594 22595 22596 22597 22598 22599 22600 22601 22602 22603 22604 22605 22606 22607 22608 22609 22610 22611 22612 22613 22614 22615 22616 22617 22618 22619 22620 22621 22622 22623 22624 22625 22626 22627 22628 22629 22630 22631 22632 22633 22634 22635 22636 22637 22638 22639 22640 22641 22642 22643 22644 22645 22646 22647 22648 22649 22650 22651 22652 22653 22654 22655 22656 22657 22658 22659 22660 22661 22662 22663 22664 22665 22666 22667 22668 22669 22670 22671 22672 22673 22674 22675 22676 22677 22678 22679 22680 22681 22682 22683 22684 22685 22686 22687 22688 22689 22690 22691 22692 22693 22694 22695 22696 22697 22698 22699 22700 22701 22702 22703 22704 22705 22706 22707 22708 22709 22710 22711 22712 22713 22714 22715 22716 22717 22718 22719 22720 22721 22722 22723 22724 22725 22726 22727 22728 22729 22730 22731 22732 22733 22734 22735 22736 22737 22738 22739 22740 22741 22742 22743 22744 22745 22746 22747 22748 22749 22750 22751 22752 22753 22754 22755 22756 22757 22758 22759 22760 22761 22762 22763 22764 22765 22766 22767 22768 22769 22770 22771 22772 22773 22774 22775 22776 22777 22778 22779 22780 22781 22782 22783 22784 22785 22786 22787 22788 22789 22790 22791 22792 22793 22794 22795 22796 22797 22798 22799 22800 22801 22802 22803 22804 22805 22806 22807 22808 22809 22810 22811 22812 22813 22814 22815 22816 22817 22818 22819 22820 22821 22822 22823 22824 22825 22826 22827 22828 22829 22830 22831 22832 22833 22834 22835 22836 22837 22838 22839 22840 22841 22842 22843 22844 22845 22846 22847 22848 22849 22850 22851 22852 22853 22854 22855 22856 22857 22858 22859 22860 22861 22862 22863 22864 22865 22866 22867 22868 22869 22870 22871 22872 22873 22874 22875 22876 22877 22878 22879 22880 22881 22882 22883 22884 22885 22886 22887 22888 22889 22890 22891 22892 22893 22894 22895 22896 22897 22898 22899 22900 22901 22902 22903 22904 22905 22906 22907 22908 22909 22910 22911 22912 22913 22914 22915 22916 22917 22918 22919 22920 22921 22922 22923 22924 22925 22926 22927 22928 22929 22930 22931 22932 22933 22934 22935 22936 22937 22938 22939 22940 22941 22942 22943 22944 22945 22946 22947 22948 22949 22950 22951 22952 22953 22954 22955 22956 22957 22958 22959 22960 22961 22962 22963 22964 22965 22966 22967 22968 22969 22970 22971 22972 22973 22974 22975 22976 22977 22978 22979 22980 22981 22982 22983 22984 22985 22986 22987 22988 22989 22990 22991 22992 22993 22994 22995 22996 22997 22998 22999 23000 23001 23002 23003 23004 23005 23006 23007 23008 23009 23010 23011 23012 23013 23014 23015 23016 23017 23018 23019 23020 23021 23022 23023 23024 23025 23026 23027 23028 23029 23030 23031 23032 23033 23034 23035 23036 23037 23038 23039 23040 23041 23042 23043 23044 23045 23046 23047 23048 23049 23050 23051 23052 23053 23054 23055 23056 23057 23058 23059 23060 23061 23062 23063 23064 23065 23066 23067 23068 23069 23070 23071 23072 23073 23074 23075 23076 23077 23078 23079 23080 23081 23082 23083 23084 23085 23086 23087 23088 23089 23090 23091 23092 23093 23094 23095 23096 23097 23098 23099 23100 23101 23102 23103 23104 23105 23106 23107 23108 23109 23110 23111 23112 23113 23114 23115 23116 23117 23118 23119 23120 23121 23122 23123 23124 23125 23126 23127 23128 23129 23130 23131 23132 23133 23134 23135 23136 23137 23138 23139 23140 23141 23142 23143 23144 23145 23146 23147 23148 23149 23150 23151 23152 23153 23154 23155 23156 23157 23158 23159 23160 23161 23162 23163 23164 23165 23166 23167 23168 23169 23170 23171 23172 23173 23174 23175 23176 23177 23178 23179 23180 23181 23182 23183 23184 23185 23186 23187 23188 23189 23190 23191 23192 23193 23194 23195 23196 23197 23198 23199 23200 23201 23202 23203 23204 23205 23206 23207 23208 23209 23210 23211 23212 23213 23214 23215 23216 23217 23218 23219 23220 23221 23222 23223 23224 23225 23226 23227 23228 23229 23230 23231 23232 23233 23234 23235 23236 23237 23238 23239 23240 23241 23242 23243 23244 23245 23246 23247 23248 23249 23250 23251 23252 23253 23254 23255 23256 23257 23258 23259 23260 23261 23262 23263 23264 23265 23266 23267 23268 23269 23270 23271 23272 23273 23274 23275 23276 23277 23278 23279 23280 23281 23282 23283 23284 23285 23286 23287 23288 23289 23290 23291 23292 23293 23294 23295 23296 23297 23298 23299 23300 23301 23302 23303 23304 23305 23306 23307 23308 23309 23310 23311 23312 23313 23314 23315 23316 23317 23318 23319 23320 23321 23322 23323 23324 23325 23326 23327 23328 23329 23330 23331 23332 23333 23334 23335 23336 23337 23338 23339 23340 23341 23342 23343 23344 23345 23346 23347 23348 23349 23350 23351 23352 23353 23354 23355 23356 23357 23358 23359 23360 23361 23362 23363 23364 23365 23366 23367 23368 23369 23370 23371 23372 23373 23374 23375 23376 23377 23378 23379 23380 23381 23382 23383 23384 23385 23386 23387 23388 23389 23390 23391 23392 23393 23394 23395 23396 23397 23398 23399 23400 23401 23402 23403 23404 23405 23406 23407 23408 23409 23410 23411 23412 23413 23414 23415 23416 23417 23418 23419 23420 23421 23422 23423 23424 23425 23426 23427 23428 23429 23430 23431 23432 23433 23434 23435 23436 23437 23438 23439 23440 23441 23442 23443 23444 23445 23446 23447 23448 23449 23450 23451 23452 23453 23454 23455 23456 23457 23458 23459 23460 23461 23462 23463 23464 23465 23466 23467 23468 23469 23470 23471 23472 23473 23474 23475 23476 23477 23478 23479 23480 23481 23482 23483 23484 23485 23486 23487 23488 23489 23490 23491 23492 23493 23494 23495 23496 23497 23498 23499 23500 23501 23502 23503 23504 23505 23506 23507 23508 23509 23510 23511 23512 23513 23514 23515 23516 23517 23518 23519 23520 23521 23522 23523 23524 23525 23526 23527 23528 23529 23530 23531 23532 23533 23534 23535 23536 23537 23538 23539 23540 23541 23542 23543 23544 23545 23546 23547 23548 23549 23550 23551 23552 23553 23554 23555 23556 23557 23558 23559 23560 23561 23562 23563 23564 23565 23566 23567 23568 23569 23570 23571 23572 23573 23574 23575 23576 23577 23578 23579 23580 23581 23582 23583 23584 23585 23586 23587 23588 23589 23590 23591 23592 23593 23594 23595 23596 23597 23598 23599 23600 23601 23602 23603 23604 23605 23606 23607 23608 23609 23610 23611 23612 23613 23614 23615 23616 23617 23618 23619 23620 23621 23622 23623 23624 23625 23626 23627 23628 23629 23630 23631 23632 23633 23634 23635 23636 23637 23638 23639 23640 23641 23642 23643 23644 23645 23646 23647 23648 23649 23650 23651 23652 23653 23654 23655 23656 23657 23658 23659 23660 23661 23662 23663 23664 23665 23666 23667 23668 23669 23670 23671 23672 23673 23674 23675 23676 23677 23678 23679 23680 23681 23682 23683 23684 23685 23686 23687 23688 23689 23690 23691 23692 23693 23694 23695 23696 23697 23698 23699 23700 23701 23702 23703 23704 23705 23706 23707 23708 23709 23710 23711 23712 23713 23714 23715 23716 23717 23718 23719 23720 23721 23722 23723 23724 23725 23726 23727 23728 23729 23730 23731 23732 23733 23734 23735 23736 23737 23738 23739 23740 23741 23742 23743 23744 23745 23746 23747 23748 23749 23750 23751 23752 23753 23754 23755 23756 23757 23758 23759 23760 23761 23762 23763 23764 23765 23766 23767 23768 23769 23770 23771 23772 23773 23774 23775 23776 23777 23778 23779 23780 23781 23782 23783 23784 23785 23786 23787 23788 23789 23790 23791 23792 23793 23794 23795 23796 23797 23798 23799 23800 23801 23802 23803 23804 23805 23806 23807 23808 23809 23810 23811 23812 23813 23814 23815 23816 23817 23818 23819 23820 23821 23822 23823 23824 23825 23826 23827 23828 23829 23830 23831 23832 23833 23834 23835 23836 23837 23838 23839 23840 23841 23842 23843 23844 23845 23846 23847 23848 23849 23850 23851 23852 23853 23854 23855 23856 23857 23858 23859 23860 23861 23862 23863 23864 23865 23866 23867 23868 23869 23870 23871 23872 23873 23874 23875 23876 23877 23878 23879 23880 23881 23882 23883 23884 23885 23886 23887 23888 23889 23890 23891 23892 23893 23894 23895 23896 23897 23898 23899 23900 23901 23902 23903 23904 23905 23906 23907 23908 23909 23910 23911 23912 23913 23914 23915 23916 23917 23918 23919 23920 23921 23922 23923 23924 23925 23926 23927 23928 23929 23930 23931 23932 23933 23934 23935 23936 23937 23938 23939 23940 23941 23942 23943 23944 23945 23946 23947 23948 23949 23950 23951 23952 23953 23954 23955 23956 23957 23958 23959 23960 23961 23962 23963 23964 23965 23966 23967 23968 23969 23970 23971 23972 23973 23974 23975 23976 23977 23978 23979 23980 23981 23982 23983 23984 23985 23986 23987 23988 23989 23990 23991 23992 23993 23994 23995 23996 23997 23998 23999 24000 24001 24002 24003 24004 24005 24006 24007 24008 24009 24010 24011 24012 24013 24014 24015 24016 24017 24018 24019 24020 24021 24022 24023 24024 24025 24026 24027 24028 24029 24030 24031 24032 24033 24034 24035 24036 24037 24038 24039 24040 24041 24042 24043 24044 24045 24046 24047 24048 24049 24050 24051 24052 24053 24054 24055 24056 24057 24058 24059 24060 24061 24062 24063 24064 24065 24066 24067 24068 24069 24070 24071 24072 24073 24074 24075 24076 24077 24078 24079 24080 24081 24082 24083 24084 24085 24086 24087 24088 24089 24090 24091 24092 24093 24094 24095 24096 24097 24098 24099 24100 24101 24102 24103 24104 24105 24106 24107 24108 24109 24110 24111 24112 24113 24114 24115 24116 24117 24118 24119 24120 24121 24122 24123 24124 24125 24126 24127 24128 24129 24130 24131 24132 24133 24134 24135 24136 24137 24138 24139 24140 24141 24142 24143 24144 24145 24146 24147 24148 24149 24150 24151 24152 24153 24154 24155 24156 24157 24158 24159 24160 24161 24162 24163 24164 24165 24166 24167 24168 24169 24170 24171 24172 24173 24174 24175 24176 24177 24178 24179 24180 24181 24182 24183 24184 24185 24186 24187 24188 24189 24190 24191 24192 24193 24194 24195 24196 24197 24198 24199 24200 24201 24202 24203 24204 24205 24206 24207 24208 24209 24210 24211 24212 24213 24214 24215 24216 24217 24218 24219 24220 24221 24222 24223 24224 24225 24226 24227 24228 24229 24230 24231 24232 24233 24234 24235 24236 24237 24238 24239 24240 24241 24242 24243 24244 24245 24246 24247 24248 24249 24250 24251 24252 24253 24254 24255 24256 24257 24258 24259 24260 24261 24262 24263 24264 24265 24266 24267 24268 24269 24270 24271 24272 24273 24274 24275 24276 24277 24278 24279 24280 24281 24282 24283 24284 24285 24286 24287 24288 24289 24290 24291 24292 24293 24294 24295 24296 24297 24298 24299 24300 24301 24302 24303 24304 24305 24306 24307 24308 24309 24310 24311 24312 24313 24314 24315 24316 24317 24318 24319 24320 24321 24322 24323 24324 24325 24326 24327 24328 24329 24330 24331 24332 24333 24334 24335 24336 24337 24338 24339 24340 24341 24342 24343 24344 24345 24346 24347 24348 24349 24350 24351 24352 24353 24354 24355 24356 24357 24358 24359 24360 24361 24362 24363 24364 24365 24366 24367 24368 24369 24370 24371 24372 24373 24374 24375 24376 24377 24378 24379 24380 24381 24382 24383 24384 24385 24386 24387 24388 24389 24390 24391 24392 24393 24394 24395 24396 24397 24398 24399 24400 24401 24402 24403 24404 24405 24406 24407 24408 24409 24410 24411 24412 24413 24414 24415 24416 24417 24418 24419 24420 24421 24422 24423 24424 24425 24426 24427 24428 24429 24430 24431 24432 24433 24434 24435 24436 24437 24438 24439 24440 24441 24442 24443 24444 24445 24446 24447 24448 24449 24450 24451 24452 24453 24454 24455 24456 24457 24458 24459 24460 24461 24462 24463 24464 24465 24466 24467 24468 24469 24470 24471 24472 24473 24474 24475 24476 24477 24478 24479 24480 24481 24482 24483 24484 24485 24486 24487 24488 24489 24490 24491 24492 24493 24494 24495 24496 24497 24498 24499 24500 24501 24502 24503 24504 24505 24506 24507 24508 24509 24510 24511 24512 24513 24514 24515 24516 24517 24518 24519 24520 24521 24522 24523 24524 24525 24526 24527 24528 24529 24530 24531 24532 24533 24534 24535 24536 24537 24538 24539 24540 24541 24542 24543 24544 24545 24546 24547 24548 24549 24550 24551 24552 24553 24554 24555 24556 24557 24558 24559 24560 24561 24562 24563 24564 24565 24566 24567 24568 24569 24570 24571 24572 24573 24574 24575 24576 24577 24578 24579 24580 24581 24582 24583 24584 24585 24586 24587 24588 24589 24590 24591 24592 24593 24594 24595 24596 24597 24598 24599 24600 24601 24602 24603 24604 24605 24606 24607 24608 24609 24610 24611 24612 24613 24614 24615 24616 24617 24618 24619 24620 24621 24622 24623 24624 24625 24626 24627 24628 24629 24630 24631 24632 24633 24634 24635 24636 24637 24638 24639 24640 24641 24642 24643 24644 24645 24646 24647 24648 24649 24650 24651 24652 24653 24654 24655 24656 24657 24658 24659 24660 24661 24662 24663 24664 24665 24666 24667 24668 24669 24670 24671 24672 24673 24674 24675 24676 24677 24678 24679 24680 24681 24682 24683 24684 24685 24686 24687 24688 24689 24690 24691 24692 24693 24694 24695 24696 24697 24698 24699 24700 24701 24702 24703 24704 24705 24706 24707 24708 24709 24710 24711 24712 24713 24714 24715 24716 24717 24718 24719 24720 24721 24722 24723 24724 24725 24726 24727 24728 24729 24730 24731 24732 24733 24734 24735 24736 24737 24738 24739 24740 24741 24742 24743 24744 24745 24746 24747 24748 24749 24750 24751 24752 24753 24754 24755 24756 24757 24758 24759 24760 24761 24762 24763 24764 24765 24766 24767 24768 24769 24770 24771 24772 24773 24774 24775 24776 24777 24778 24779 24780 24781 24782 24783 24784 24785 24786 24787 24788 24789 24790 24791 24792 24793 24794 24795 24796 24797 24798 24799 24800 24801 24802 24803 24804 24805 24806 24807 24808 24809 24810 24811 24812 24813 24814 24815 24816 24817 24818 24819 24820 24821 24822 24823 24824 24825 24826 24827 24828 24829 24830 24831 24832 24833 24834 24835 24836 24837 24838 24839 24840 24841 24842 24843 24844 24845 24846 24847 24848 24849 24850 24851 24852 24853 24854 24855 24856 24857 24858 24859 24860 24861 24862 24863 24864 24865 24866 24867 24868 24869 24870 24871 24872 24873 24874 24875 24876 24877 24878 24879 24880 24881 24882 24883 24884 24885 24886 24887 24888 24889 24890 24891 24892 24893 24894 24895 24896 24897 24898 24899 24900 24901 24902 24903 24904 24905 24906 24907 24908 24909 24910 24911 24912 24913 24914 24915 24916 24917 24918 24919 24920 24921 24922 24923 24924 24925 24926 24927 24928 24929 24930 24931 24932 24933 24934 24935 24936 24937 24938 24939 24940 24941 24942 24943 24944 24945 24946 24947 24948 24949 24950 24951 24952 24953 24954 24955 24956 24957 24958 24959 24960 24961 24962 24963 24964 24965 24966 24967 24968 24969 24970 24971 24972 24973 24974 24975 24976 24977 24978 24979 24980 24981 24982 24983 24984 24985 24986 24987 24988 24989 24990 24991 24992 24993 24994 24995 24996 24997 24998 24999 25000 25001 25002 25003 25004 25005 25006 25007 25008 25009 25010 25011 25012 25013 25014 25015 25016 25017 25018 25019 25020 25021 25022 25023 25024 25025 25026 25027 25028 25029 25030 25031 25032 25033 25034 25035 25036 25037 25038 25039 25040 25041 25042 25043 25044 25045 25046 25047 25048 25049 25050 25051 25052 25053 25054 25055 25056 25057 25058 25059 25060 25061 25062 25063 25064 25065 25066 25067 25068 25069 25070 25071 25072 25073 25074 25075 25076 25077 25078 25079 25080 25081 25082 25083 25084 25085 25086 25087 25088 25089 25090 25091 25092 25093 25094 25095 25096 25097 25098 25099 25100 25101 25102 25103 25104 25105 25106 25107 25108 25109 25110 25111 25112 25113 25114 25115 25116 25117 25118 25119 25120 25121 25122 25123 25124 25125 25126 25127 25128 25129 25130 25131 25132 25133 25134 25135 25136 25137 25138 25139 25140 25141 25142 25143 25144 25145 25146 25147 25148 25149 25150 25151 25152 25153 25154 25155 25156 25157 25158 25159 25160 25161 25162 25163 25164 25165 25166 25167 25168 25169 25170 25171 25172 25173 25174 25175 25176 25177 25178 25179 25180 25181 25182 25183 25184 25185 25186 25187 25188 25189 25190 25191 25192 25193 25194 25195 25196 25197 25198 25199 25200 25201 25202 25203 25204 25205 25206 25207 25208 25209 25210 25211 25212 25213 25214 25215 25216 25217 25218 25219 25220 25221 25222 25223 25224 25225 25226 25227 25228 25229 25230 25231 25232 25233 25234 25235 25236 25237 25238 25239 25240 25241 25242 25243 25244 25245 25246 25247 25248 25249 25250 25251 25252 25253 25254 25255 25256 25257 25258 25259 25260 25261 25262 25263 25264 25265 25266 25267 25268 25269 25270 25271 25272 25273 25274 25275 25276 25277 25278 25279 25280 25281 25282 25283 25284 25285 25286 25287 25288 25289 25290 25291 25292 25293 25294 25295 25296 25297 25298 25299 25300 25301 25302 25303 25304 25305 25306 25307 25308 25309 25310 25311 25312 25313 25314 25315 25316 25317 25318 25319 25320 25321 25322 25323 25324 25325 25326 25327 25328 25329 25330 25331 25332 25333 25334 25335 25336 25337 25338 25339 25340 25341 25342 25343 25344 25345 25346 25347 25348 25349 25350 25351 25352 25353 25354 25355 25356 25357 25358 25359 25360 25361 25362 25363 25364 25365 25366 25367 25368 25369 25370 25371 25372 25373 25374 25375 25376 25377 25378 25379 25380 25381 25382 25383 25384 25385 25386 25387 25388 25389 25390 25391 25392 25393 25394 25395 25396 25397 25398 25399 25400 25401 25402 25403 25404 25405 25406 25407 25408 25409 25410 25411 25412 25413 25414 25415 25416 25417 25418 25419 25420 25421 25422 25423 25424 25425 25426 25427 25428 25429 25430 25431 25432 25433 25434 25435 25436 25437 25438 25439 25440 25441 25442 25443 25444 25445 25446 25447 25448 25449 25450 25451 25452 25453 25454 25455 25456 25457 25458 25459 25460 25461 25462 25463 25464 25465 25466 25467 25468 25469 25470 25471 25472 25473 25474 25475 25476 25477 25478 25479 25480 25481 25482 25483 25484 25485 25486 25487 25488 25489 25490 25491 25492 25493 25494 25495 25496 25497 25498 25499 25500 25501 25502 25503 25504 25505 25506 25507 25508 25509 25510 25511 25512 25513 25514 25515 25516 25517 25518 25519 25520 25521 25522 25523 25524 25525 25526 25527 25528 25529 25530 25531 25532 25533 25534 25535 25536 25537 25538 25539 25540 25541 25542 25543 25544 25545 25546 25547 25548 25549 25550 25551 25552 25553 25554 25555 25556 25557 25558 25559 25560 25561 25562 25563 25564 25565 25566 25567 25568 25569 25570 25571 25572 25573 25574 25575 25576 25577 25578 25579 25580 25581 25582 25583 25584 25585 25586 25587 25588 25589 25590 25591 25592 25593 25594 25595 25596 25597 25598 25599 25600 25601 25602 25603 25604 25605 25606 25607 25608 25609 25610 25611 25612 25613 25614 25615 25616 25617 25618 25619 25620 25621 25622 25623 25624 25625 25626 25627 25628 25629 25630 25631 25632 25633 25634 25635 25636 25637 25638 25639 25640 25641 25642 25643 25644 25645 25646 25647 25648 25649 25650 25651 25652 25653 25654 25655 25656 25657 25658 25659 25660 25661 25662 25663 25664 25665 25666 25667 25668 25669 25670 25671 25672 25673 25674 25675 25676 25677 25678 25679 25680 25681 25682 25683 25684 25685 25686 25687 25688 25689 25690 25691 25692 25693 25694 25695 25696 25697 25698 25699 25700 25701 25702 25703 25704 25705 25706 25707 25708 25709 25710 25711 25712 25713 25714 25715 25716 25717 25718 25719 25720 25721 25722 25723 25724 25725 25726 25727 25728 25729 25730 25731 25732 25733 25734 25735 25736 25737 25738 25739 25740 25741 25742 25743 25744 25745 25746 25747 25748 25749 25750 25751 25752 25753 25754 25755 25756 25757 25758 25759 25760 25761 25762 25763 25764 25765 25766 25767 25768 25769 25770 25771 25772 25773 25774 25775 25776 25777 25778 25779 25780 25781 25782 25783 25784 25785 25786 25787 25788 25789 25790 25791 25792 25793 25794 25795 25796 25797 25798 25799 25800 25801 25802 25803 25804 25805 25806 25807 25808 25809 25810 25811 25812 25813 25814 25815 25816 25817 25818 25819 25820 25821 25822 25823 25824 25825 25826 25827 25828 25829 25830 25831 25832 25833 25834 25835 25836 25837 25838 25839 25840 25841 25842 25843 25844 25845 25846 25847 25848 25849 25850 25851 25852 25853 25854 25855 25856 25857 25858 25859 25860 25861 25862 25863 25864 25865 25866 25867 25868 25869 25870 25871 25872 25873 25874 25875 25876 25877 25878 25879 25880 25881 25882 25883 25884 25885 25886 25887 25888 25889 25890 25891 25892 25893 25894 25895 25896 25897 25898 25899 25900 25901 25902 25903 25904 25905 25906 25907 25908 25909 25910 25911 25912 25913 25914 25915 25916 25917 25918 25919 25920 25921 25922 25923 25924 25925 25926 25927 25928 25929 25930 25931 25932 25933 25934 25935 25936 25937 25938 25939 25940 25941 25942 25943 25944 25945 25946 25947 
/* Subroutines used for code generation on IA-32.
   Copyright (C) 1988-2023 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.

GCC is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#define IN_TARGET_CODE 1

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "rtl.h"
#include "tree.h"
#include "memmodel.h"
#include "gimple.h"
#include "cfghooks.h"
#include "cfgloop.h"
#include "df.h"
#include "tm_p.h"
#include "stringpool.h"
#include "expmed.h"
#include "optabs.h"
#include "regs.h"
#include "emit-rtl.h"
#include "recog.h"
#include "cgraph.h"
#include "diagnostic.h"
#include "cfgbuild.h"
#include "alias.h"
#include "fold-const.h"
#include "attribs.h"
#include "calls.h"
#include "stor-layout.h"
#include "varasm.h"
#include "output.h"
#include "insn-attr.h"
#include "flags.h"
#include "except.h"
#include "explow.h"
#include "expr.h"
#include "cfgrtl.h"
#include "common/common-target.h"
#include "langhooks.h"
#include "reload.h"
#include "gimplify.h"
#include "dwarf2.h"
#include "tm-constrs.h"
#include "cselib.h"
#include "sched-int.h"
#include "opts.h"
#include "tree-pass.h"
#include "context.h"
#include "pass_manager.h"
#include "target-globals.h"
#include "gimple-iterator.h"
#include "gimple-fold.h"
#include "tree-vectorizer.h"
#include "shrink-wrap.h"
#include "builtins.h"
#include "rtl-iter.h"
#include "tree-iterator.h"
#include "dbgcnt.h"
#include "case-cfn-macros.h"
#include "dojump.h"
#include "fold-const-call.h"
#include "tree-vrp.h"
#include "tree-ssanames.h"
#include "selftest.h"
#include "selftest-rtl.h"
#include "print-rtl.h"
#include "intl.h"
#include "ifcvt.h"
#include "symbol-summary.h"
#include "ipa-prop.h"
#include "ipa-fnsummary.h"
#include "wide-int-bitmask.h"
#include "tree-vector-builder.h"
#include "debug.h"
#include "dwarf2out.h"
#include "i386-options.h"
#include "i386-builtins.h"
#include "i386-expand.h"
#include "i386-features.h"
#include "function-abi.h"
#include "rtl-error.h"

/* This file should be included last.  */
#include "target-def.h"

static rtx legitimize_dllimport_symbol (rtx, bool);
static rtx legitimize_pe_coff_extern_decl (rtx, bool);
static void ix86_print_operand_address_as (FILE *, rtx, addr_space_t, bool);
static void ix86_emit_restore_reg_using_pop (rtx);


#ifndef CHECK_STACK_LIMIT
#define CHECK_STACK_LIMIT (-1)
#endif

/* Return index of given mode in mult and division cost tables.  */
#define MODE_INDEX(mode)					\
  ((mode) == QImode ? 0						\
   : (mode) == HImode ? 1					\
   : (mode) == SImode ? 2					\
   : (mode) == DImode ? 3					\
   : 4)


/* Set by -mtune.  */
const struct processor_costs *ix86_tune_cost = NULL;

/* Set by -mtune or -Os.  */
const struct processor_costs *ix86_cost = NULL;

/* In case the average insn count for single function invocation is
   lower than this constant, emit fast (but longer) prologue and
   epilogue code.  */
#define FAST_PROLOGUE_INSN_COUNT 20

/* Names for 8 (low), 8 (high), and 16-bit registers, respectively.  */
static const char *const qi_reg_name[] = QI_REGISTER_NAMES;
static const char *const qi_high_reg_name[] = QI_HIGH_REGISTER_NAMES;
static const char *const hi_reg_name[] = HI_REGISTER_NAMES;

/* Array of the smallest class containing reg number REGNO, indexed by
   REGNO.  Used by REGNO_REG_CLASS in i386.h.  */

enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
{
  /* ax, dx, cx, bx */
  AREG, DREG, CREG, BREG,
  /* si, di, bp, sp */
  SIREG, DIREG, NON_Q_REGS, NON_Q_REGS,
  /* FP registers */
  FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS,
  FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS,
  /* arg pointer, flags, fpsr, frame */
  NON_Q_REGS, NO_REGS, NO_REGS, NON_Q_REGS,
  /* SSE registers */
  SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS,
  SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
  /* MMX registers */
  MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
  MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
  /* REX registers */
  GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
  GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
  /* SSE REX registers */
  SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
  SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
  /* AVX-512 SSE registers */
  ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
  ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
  ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
  ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
  /* Mask registers.  */
  ALL_MASK_REGS, MASK_REGS, MASK_REGS, MASK_REGS,
  MASK_REGS, MASK_REGS, MASK_REGS, MASK_REGS,
  /* REX2 registers */
  GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
  GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
  GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
  GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
};

/* The "default" register map used in 32bit mode.  */

int const debugger_register_map[FIRST_PSEUDO_REGISTER] =
{
  /* general regs */
  0, 2, 1, 3, 6, 7, 4, 5,
  /* fp regs */
  12, 13, 14, 15, 16, 17, 18, 19,
  /* arg, flags, fpsr, frame */
  IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
  IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
  /* SSE */
  21, 22, 23, 24, 25, 26, 27, 28,
  /* MMX */
  29, 30, 31, 32, 33, 34, 35, 36,
  /* extended integer registers */
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  /* extended sse registers */
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  /* AVX-512 registers 16-23 */
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  /* AVX-512 registers 24-31 */
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  /* Mask registers */
  93, 94, 95, 96, 97, 98, 99, 100
};

/* The "default" register map used in 64bit mode.  */

int const debugger64_register_map[FIRST_PSEUDO_REGISTER] =
{
  /* general regs */
  0, 1, 2, 3, 4, 5, 6, 7,
  /* fp regs */
  33, 34, 35, 36, 37, 38, 39, 40,
  /* arg, flags, fpsr, frame */
  IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
  IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
  /* SSE */
  17, 18, 19, 20, 21, 22, 23, 24,
  /* MMX */
  41, 42, 43, 44, 45, 46, 47, 48,
  /* extended integer registers */
  8, 9, 10, 11, 12, 13, 14, 15,
  /* extended SSE registers */
  25, 26, 27, 28, 29, 30, 31, 32,
  /* AVX-512 registers 16-23 */
  67, 68, 69, 70, 71, 72, 73, 74,
  /* AVX-512 registers 24-31 */
  75, 76, 77, 78, 79, 80, 81, 82,
  /* Mask registers */
  118, 119, 120, 121, 122, 123, 124, 125,
  /* rex2 extend interger registers */
  130, 131, 132, 133, 134, 135, 136, 137,
  138, 139, 140, 141, 142, 143, 144, 145
};

/* Define the register numbers to be used in Dwarf debugging information.
   The SVR4 reference port C compiler uses the following register numbers
   in its Dwarf output code:
	0 for %eax (gcc regno = 0)
	1 for %ecx (gcc regno = 2)
	2 for %edx (gcc regno = 1)
	3 for %ebx (gcc regno = 3)
	4 for %esp (gcc regno = 7)
	5 for %ebp (gcc regno = 6)
	6 for %esi (gcc regno = 4)
	7 for %edi (gcc regno = 5)
   The following three DWARF register numbers are never generated by
   the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
   believed these numbers have these meanings.
	8  for %eip    (no gcc equivalent)
	9  for %eflags (gcc regno = 17)
	10 for %trapno (no gcc equivalent)
   It is not at all clear how we should number the FP stack registers
   for the x86 architecture.  If the version of SDB on x86/svr4 were
   a bit less brain dead with respect to floating-point then we would
   have a precedent to follow with respect to DWARF register numbers
   for x86 FP registers, but the SDB on x86/svr4 was so completely
   broken with respect to FP registers that it is hardly worth thinking
   of it as something to strive for compatibility with.
   The version of x86/svr4 SDB I had does (partially)
   seem to believe that DWARF register number 11 is associated with
   the x86 register %st(0), but that's about all.  Higher DWARF
   register numbers don't seem to be associated with anything in
   particular, and even for DWARF regno 11, SDB only seemed to under-
   stand that it should say that a variable lives in %st(0) (when
   asked via an `=' command) if we said it was in DWARF regno 11,
   but SDB still printed garbage when asked for the value of the
   variable in question (via a `/' command).
   (Also note that the labels SDB printed for various FP stack regs
   when doing an `x' command were all wrong.)
   Note that these problems generally don't affect the native SVR4
   C compiler because it doesn't allow the use of -O with -g and
   because when it is *not* optimizing, it allocates a memory
   location for each floating-point variable, and the memory
   location is what gets described in the DWARF AT_location
   attribute for the variable in question.
   Regardless of the severe mental illness of the x86/svr4 SDB, we
   do something sensible here and we use the following DWARF
   register numbers.  Note that these are all stack-top-relative
   numbers.
	11 for %st(0) (gcc regno = 8)
	12 for %st(1) (gcc regno = 9)
	13 for %st(2) (gcc regno = 10)
	14 for %st(3) (gcc regno = 11)
	15 for %st(4) (gcc regno = 12)
	16 for %st(5) (gcc regno = 13)
	17 for %st(6) (gcc regno = 14)
	18 for %st(7) (gcc regno = 15)
*/
int const svr4_debugger_register_map[FIRST_PSEUDO_REGISTER] =
{
  /* general regs */
  0, 2, 1, 3, 6, 7, 5, 4,
  /* fp regs */
  11, 12, 13, 14, 15, 16, 17, 18,
  /* arg, flags, fpsr, frame */
  IGNORED_DWARF_REGNUM, 9,
  IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
  /* SSE registers */
  21, 22, 23, 24, 25, 26, 27, 28,
  /* MMX registers */
  29, 30, 31, 32, 33, 34, 35, 36,
  /* extended integer registers */
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  /* extended sse registers */
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  /* AVX-512 registers 16-23 */
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  /* AVX-512 registers 24-31 */
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
  /* Mask registers */
  93, 94, 95, 96, 97, 98, 99, 100
};

/* Define parameter passing and return registers.  */

static int const x86_64_int_parameter_registers[6] =
{
  DI_REG, SI_REG, DX_REG, CX_REG, R8_REG, R9_REG
};

static int const x86_64_ms_abi_int_parameter_registers[4] =
{
  CX_REG, DX_REG, R8_REG, R9_REG
};

static int const x86_64_int_return_registers[4] =
{
  AX_REG, DX_REG, DI_REG, SI_REG
};

/* Define the structure for the machine field in struct function.  */

struct GTY(()) stack_local_entry {
  unsigned short mode;
  unsigned short n;
  rtx rtl;
  struct stack_local_entry *next;
};

/* Which cpu are we scheduling for.  */
enum attr_cpu ix86_schedule;

/* Which cpu are we optimizing for.  */
enum processor_type ix86_tune;

/* Which instruction set architecture to use.  */
enum processor_type ix86_arch;

/* True if processor has SSE prefetch instruction.  */
unsigned char ix86_prefetch_sse;

/* Preferred alignment for stack boundary in bits.  */
unsigned int ix86_preferred_stack_boundary;

/* Alignment for incoming stack boundary in bits specified at
   command line.  */
unsigned int ix86_user_incoming_stack_boundary;

/* Default alignment for incoming stack boundary in bits.  */
unsigned int ix86_default_incoming_stack_boundary;

/* Alignment for incoming stack boundary in bits.  */
unsigned int ix86_incoming_stack_boundary;

/* True if there is no direct access to extern symbols.  */
bool ix86_has_no_direct_extern_access;

/* Calling abi specific va_list type nodes.  */
tree sysv_va_list_type_node;
tree ms_va_list_type_node;

/* Prefix built by ASM_GENERATE_INTERNAL_LABEL.  */
char internal_label_prefix[16];
int internal_label_prefix_len;

/* Fence to use after loop using movnt.  */
tree x86_mfence;

/* Register class used for passing given 64bit part of the argument.
   These represent classes as documented by the PS ABI, with the exception
   of SSESF, SSEDF classes, that are basically SSE class, just gcc will
   use SF or DFmode move instead of DImode to avoid reformatting penalties.

   Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
   whenever possible (upper half does contain padding).  */
enum x86_64_reg_class
  {
    X86_64_NO_CLASS,
    X86_64_INTEGER_CLASS,
    X86_64_INTEGERSI_CLASS,
    X86_64_SSE_CLASS,
    X86_64_SSEHF_CLASS,
    X86_64_SSESF_CLASS,
    X86_64_SSEDF_CLASS,
    X86_64_SSEUP_CLASS,
    X86_64_X87_CLASS,
    X86_64_X87UP_CLASS,
    X86_64_COMPLEX_X87_CLASS,
    X86_64_MEMORY_CLASS
  };

#define MAX_CLASSES 8

/* Table of constants used by fldpi, fldln2, etc....  */
static REAL_VALUE_TYPE ext_80387_constants_table [5];
static bool ext_80387_constants_init;


static rtx ix86_function_value (const_tree, const_tree, bool);
static bool ix86_function_value_regno_p (const unsigned int);
static unsigned int ix86_function_arg_boundary (machine_mode,
						const_tree);
static rtx ix86_static_chain (const_tree, bool);
static int ix86_function_regparm (const_tree, const_tree);
static void ix86_compute_frame_layout (void);
static tree ix86_canonical_va_list_type (tree);
static unsigned int split_stack_prologue_scratch_regno (void);
static bool i386_asm_output_addr_const_extra (FILE *, rtx);

static bool ix86_can_inline_p (tree, tree);
static unsigned int ix86_minimum_incoming_stack_boundary (bool);


/* Whether -mtune= or -march= were specified */
int ix86_tune_defaulted;
int ix86_arch_specified;

/* Return true if a red-zone is in use.  We can't use red-zone when
   there are local indirect jumps, like "indirect_jump" or "tablejump",
   which jumps to another place in the function, since "call" in the
   indirect thunk pushes the return address onto stack, destroying
   red-zone.

   TODO: If we can reserve the first 2 WORDs, for PUSH and, another
   for CALL, in red-zone, we can allow local indirect jumps with
   indirect thunk.  */

bool
ix86_using_red_zone (void)
{
  return (TARGET_RED_ZONE
	  && !TARGET_64BIT_MS_ABI
	  && (!cfun->machine->has_local_indirect_jump
	      || cfun->machine->indirect_branch_type == indirect_branch_keep));
}

/* Return true, if profiling code should be emitted before
   prologue. Otherwise it returns false.
   Note: For x86 with "hotfix" it is sorried.  */
static bool
ix86_profile_before_prologue (void)
{
  return flag_fentry != 0;
}

/* Update register usage after having seen the compiler flags.  */

static void
ix86_conditional_register_usage (void)
{
  int i, c_mask;

  /* If there are no caller-saved registers, preserve all registers.
     except fixed_regs and registers used for function return value
     since aggregate_value_p checks call_used_regs[regno] on return
     value.  */
  if (cfun && cfun->machine->no_caller_saved_registers)
    for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
      if (!fixed_regs[i] && !ix86_function_value_regno_p (i))
	call_used_regs[i] = 0;

  /* For 32-bit targets, disable the REX registers.  */
  if (! TARGET_64BIT)
    {
      for (i = FIRST_REX_INT_REG; i <= LAST_REX_INT_REG; i++)
	CLEAR_HARD_REG_BIT (accessible_reg_set, i);
      for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
	CLEAR_HARD_REG_BIT (accessible_reg_set, i);
      for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
	CLEAR_HARD_REG_BIT (accessible_reg_set, i);
    }

  /*  See the definition of CALL_USED_REGISTERS in i386.h.  */
  c_mask = CALL_USED_REGISTERS_MASK (TARGET_64BIT_MS_ABI);
  
  CLEAR_HARD_REG_SET (reg_class_contents[(int)CLOBBERED_REGS]);

  for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
    {
      /* Set/reset conditionally defined registers from
	 CALL_USED_REGISTERS initializer.  */
      if (call_used_regs[i] > 1)
	call_used_regs[i] = !!(call_used_regs[i] & c_mask);

      /* Calculate registers of CLOBBERED_REGS register set
	 as call used registers from GENERAL_REGS register set.  */
      if (TEST_HARD_REG_BIT (reg_class_contents[(int)GENERAL_REGS], i)
	  && call_used_regs[i])
	SET_HARD_REG_BIT (reg_class_contents[(int)CLOBBERED_REGS], i);
    }

  /* If MMX is disabled, disable the registers.  */
  if (! TARGET_MMX)
    accessible_reg_set &= ~reg_class_contents[MMX_REGS];

  /* If SSE is disabled, disable the registers.  */
  if (! TARGET_SSE)
    accessible_reg_set &= ~reg_class_contents[ALL_SSE_REGS];

  /* If the FPU is disabled, disable the registers.  */
  if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
    accessible_reg_set &= ~reg_class_contents[FLOAT_REGS];

  /* If AVX512F is disabled, disable the registers.  */
  if (! TARGET_AVX512F)
    {
      for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
	CLEAR_HARD_REG_BIT (accessible_reg_set, i);

      accessible_reg_set &= ~reg_class_contents[ALL_MASK_REGS];
    }

  /* If APX is disabled, disable the registers.  */
  if (! (TARGET_APX_EGPR && TARGET_64BIT))
    {
      for (i = FIRST_REX2_INT_REG; i <= LAST_REX2_INT_REG; i++)
	CLEAR_HARD_REG_BIT (accessible_reg_set, i);
    }
}

/* Canonicalize a comparison from one we don't have to one we do have.  */

static void
ix86_canonicalize_comparison (int *code, rtx *op0, rtx *op1,
			      bool op0_preserve_value)
{
  /* The order of operands in x87 ficom compare is forced by combine in
     simplify_comparison () function. Float operator is treated as RTX_OBJ
     with a precedence over other operators and is always put in the first
     place. Swap condition and operands to match ficom instruction.  */
  if (!op0_preserve_value
      && GET_CODE (*op0) == FLOAT && MEM_P (XEXP (*op0, 0)) && REG_P (*op1))
    {
      enum rtx_code scode = swap_condition ((enum rtx_code) *code);

      /* We are called only for compares that are split to SAHF instruction.
	 Ensure that we have setcc/jcc insn for the swapped condition.  */
      if (ix86_fp_compare_code_to_integer (scode) != UNKNOWN)
	{
	  std::swap (*op0, *op1);
	  *code = (int) scode;
	}
    }
}


/* Hook to determine if one function can safely inline another.  */

static bool
ix86_can_inline_p (tree caller, tree callee)
{
  tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller);
  tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee);

  /* Changes of those flags can be tolerated for always inlines. Lets hope
     user knows what he is doing.  */
  unsigned HOST_WIDE_INT always_inline_safe_mask
	 = (MASK_USE_8BIT_IDIV | MASK_ACCUMULATE_OUTGOING_ARGS
	    | MASK_NO_ALIGN_STRINGOPS | MASK_AVX256_SPLIT_UNALIGNED_LOAD
	    | MASK_AVX256_SPLIT_UNALIGNED_STORE | MASK_CLD
	    | MASK_NO_FANCY_MATH_387 | MASK_IEEE_FP | MASK_INLINE_ALL_STRINGOPS
	    | MASK_INLINE_STRINGOPS_DYNAMICALLY | MASK_RECIP | MASK_STACK_PROBE
	    | MASK_STV | MASK_TLS_DIRECT_SEG_REFS | MASK_VZEROUPPER
	    | MASK_NO_PUSH_ARGS | MASK_OMIT_LEAF_FRAME_POINTER);


  if (!callee_tree)
    callee_tree = target_option_default_node;
  if (!caller_tree)
    caller_tree = target_option_default_node;
  if (callee_tree == caller_tree)
    return true;

  struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree);
  struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree);
  bool ret = false;
  bool always_inline
    = (DECL_DISREGARD_INLINE_LIMITS (callee)
       && lookup_attribute ("always_inline",
			    DECL_ATTRIBUTES (callee)));

  /* If callee only uses GPRs, ignore MASK_80387.  */
  if (TARGET_GENERAL_REGS_ONLY_P (callee_opts->x_ix86_target_flags))
    always_inline_safe_mask |= MASK_80387;

  cgraph_node *callee_node = cgraph_node::get (callee);
  /* Callee's isa options should be a subset of the caller's, i.e. a SSE4
     function can inline a SSE2 function but a SSE2 function can't inline
     a SSE4 function.  */
  if (((caller_opts->x_ix86_isa_flags & callee_opts->x_ix86_isa_flags)
       != callee_opts->x_ix86_isa_flags)
      || ((caller_opts->x_ix86_isa_flags2 & callee_opts->x_ix86_isa_flags2)
	  != callee_opts->x_ix86_isa_flags2))
    ret = false;

  /* See if we have the same non-isa options.  */
  else if ((!always_inline
	    && caller_opts->x_target_flags != callee_opts->x_target_flags)
	   || (caller_opts->x_target_flags & ~always_inline_safe_mask)
	       != (callee_opts->x_target_flags & ~always_inline_safe_mask))
    ret = false;

  else if (caller_opts->x_ix86_fpmath != callee_opts->x_ix86_fpmath
	   /* If the calle doesn't use FP expressions differences in
	      ix86_fpmath can be ignored.  We are called from FEs
	      for multi-versioning call optimization, so beware of
	      ipa_fn_summaries not available.  */
	   && (! ipa_fn_summaries
	       || ipa_fn_summaries->get (callee_node) == NULL
	       || ipa_fn_summaries->get (callee_node)->fp_expressions))
    ret = false;

  /* At this point we cannot identify whether arch or tune setting
     comes from target attribute or not. So the most conservative way
     is to allow the callee that uses default arch and tune string to
     be inlined.  */
  else if (!strcmp (callee_opts->x_ix86_arch_string, "x86-64")
	   && !strcmp (callee_opts->x_ix86_tune_string, "generic"))
    ret = true;

  /* See if arch, tune, etc. are the same. As previous ISA flags already
     checks if callee's ISA is subset of caller's, do not block
     always_inline attribute for callee even it has different arch. */
  else if (!always_inline && caller_opts->arch != callee_opts->arch)
    ret = false;

  else if (!always_inline && caller_opts->tune != callee_opts->tune)
    ret = false;

  else if (!always_inline
	   && caller_opts->branch_cost != callee_opts->branch_cost)
    ret = false;

  else
    ret = true;

  return ret;
}

/* Return true if this goes in large data/bss.  */

static bool
ix86_in_large_data_p (tree exp)
{
  if (ix86_cmodel != CM_MEDIUM && ix86_cmodel != CM_MEDIUM_PIC)
    return false;

  if (exp == NULL_TREE)
    return false;

  /* Functions are never large data.  */
  if (TREE_CODE (exp) == FUNCTION_DECL)
    return false;

  /* Automatic variables are never large data.  */
  if (VAR_P (exp) && !is_global_var (exp))
    return false;

  if (VAR_P (exp) && DECL_SECTION_NAME (exp))
    {
      const char *section = DECL_SECTION_NAME (exp);
      if (strcmp (section, ".ldata") == 0
	  || strcmp (section, ".lbss") == 0)
	return true;
      return false;
    }
  else
    {
      HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));

      /* If this is an incomplete type with size 0, then we can't put it
	 in data because it might be too big when completed.  Also,
	 int_size_in_bytes returns -1 if size can vary or is larger than
	 an integer in which case also it is safer to assume that it goes in
	 large data.  */
      if (size <= 0 || size > ix86_section_threshold)
	return true;
    }

  return false;
}

/* i386-specific section flag to mark large sections.  */
#define SECTION_LARGE SECTION_MACH_DEP

/* Switch to the appropriate section for output of DECL.
   DECL is either a `VAR_DECL' node or a constant of some sort.
   RELOC indicates whether forming the initial value of DECL requires
   link-time relocations.  */

ATTRIBUTE_UNUSED static section *
x86_64_elf_select_section (tree decl, int reloc,
			   unsigned HOST_WIDE_INT align)
{
  if (ix86_in_large_data_p (decl))
    {
      const char *sname = NULL;
      unsigned int flags = SECTION_WRITE | SECTION_LARGE;
      switch (categorize_decl_for_section (decl, reloc))
	{
	case SECCAT_DATA:
	  sname = ".ldata";
	  break;
	case SECCAT_DATA_REL:
	  sname = ".ldata.rel";
	  break;
	case SECCAT_DATA_REL_LOCAL:
	  sname = ".ldata.rel.local";
	  break;
	case SECCAT_DATA_REL_RO:
	  sname = ".ldata.rel.ro";
	  break;
	case SECCAT_DATA_REL_RO_LOCAL:
	  sname = ".ldata.rel.ro.local";
	  break;
	case SECCAT_BSS:
	  sname = ".lbss";
	  flags |= SECTION_BSS;
	  break;
	case SECCAT_RODATA:
	case SECCAT_RODATA_MERGE_STR:
	case SECCAT_RODATA_MERGE_STR_INIT:
	case SECCAT_RODATA_MERGE_CONST:
	  sname = ".lrodata";
	  flags &= ~SECTION_WRITE;
	  break;
	case SECCAT_SRODATA:
	case SECCAT_SDATA:
	case SECCAT_SBSS:
	  gcc_unreachable ();
	case SECCAT_TEXT:
	case SECCAT_TDATA:
	case SECCAT_TBSS:
	  /* We don't split these for medium model.  Place them into
	     default sections and hope for best.  */
	  break;
	}
      if (sname)
	{
	  /* We might get called with string constants, but get_named_section
	     doesn't like them as they are not DECLs.  Also, we need to set
	     flags in that case.  */
	  if (!DECL_P (decl))
	    return get_section (sname, flags, NULL);
	  return get_named_section (decl, sname, reloc);
	}
    }
  return default_elf_select_section (decl, reloc, align);
}

/* Select a set of attributes for section NAME based on the properties
   of DECL and whether or not RELOC indicates that DECL's initializer
   might contain runtime relocations.  */

static unsigned int ATTRIBUTE_UNUSED
x86_64_elf_section_type_flags (tree decl, const char *name, int reloc)
{
  unsigned int flags = default_section_type_flags (decl, name, reloc);

  if (ix86_in_large_data_p (decl))
    flags |= SECTION_LARGE;

  if (decl == NULL_TREE
      && (strcmp (name, ".ldata.rel.ro") == 0
	  || strcmp (name, ".ldata.rel.ro.local") == 0))
    flags |= SECTION_RELRO;

  if (strcmp (name, ".lbss") == 0
      || startswith (name, ".lbss.")
      || startswith (name, ".gnu.linkonce.lb."))
    flags |= SECTION_BSS;

  return flags;
}

/* Build up a unique section name, expressed as a
   STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
   RELOC indicates whether the initial value of EXP requires
   link-time relocations.  */

static void ATTRIBUTE_UNUSED
x86_64_elf_unique_section (tree decl, int reloc)
{
  if (ix86_in_large_data_p (decl))
    {
      const char *prefix = NULL;
      /* We only need to use .gnu.linkonce if we don't have COMDAT groups.  */
      bool one_only = DECL_COMDAT_GROUP (decl) && !HAVE_COMDAT_GROUP;

      switch (categorize_decl_for_section (decl, reloc))
	{
	case SECCAT_DATA:
	case SECCAT_DATA_REL:
	case SECCAT_DATA_REL_LOCAL:
	case SECCAT_DATA_REL_RO:
	case SECCAT_DATA_REL_RO_LOCAL:
          prefix = one_only ? ".ld" : ".ldata";
	  break;
	case SECCAT_BSS:
          prefix = one_only ? ".lb" : ".lbss";
	  break;
	case SECCAT_RODATA:
	case SECCAT_RODATA_MERGE_STR:
	case SECCAT_RODATA_MERGE_STR_INIT:
	case SECCAT_RODATA_MERGE_CONST:
          prefix = one_only ? ".lr" : ".lrodata";
	  break;
	case SECCAT_SRODATA:
	case SECCAT_SDATA:
	case SECCAT_SBSS:
	  gcc_unreachable ();
	case SECCAT_TEXT:
	case SECCAT_TDATA:
	case SECCAT_TBSS:
	  /* We don't split these for medium model.  Place them into
	     default sections and hope for best.  */
	  break;
	}
      if (prefix)
	{
	  const char *name, *linkonce;
	  char *string;

	  name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
	  name = targetm.strip_name_encoding (name);

	  /* If we're using one_only, then there needs to be a .gnu.linkonce
     	     prefix to the section name.  */
	  linkonce = one_only ? ".gnu.linkonce" : "";

	  string = ACONCAT ((linkonce, prefix, ".", name, NULL));

	  set_decl_section_name (decl, string);
	  return;
	}
    }
  default_unique_section (decl, reloc);
}

#ifdef COMMON_ASM_OP

#ifndef LARGECOMM_SECTION_ASM_OP
#define LARGECOMM_SECTION_ASM_OP "\t.largecomm\t"
#endif

/* This says how to output assembler code to declare an
   uninitialized external linkage data object.

   For medium model x86-64 we need to use LARGECOMM_SECTION_ASM_OP opcode for
   large objects.  */
void
x86_elf_aligned_decl_common (FILE *file, tree decl,
			const char *name, unsigned HOST_WIDE_INT size,
			unsigned align)
{
  if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
      && size > (unsigned int)ix86_section_threshold)
    {
      switch_to_section (get_named_section (decl, ".lbss", 0));
      fputs (LARGECOMM_SECTION_ASM_OP, file);
    }
  else
    fputs (COMMON_ASM_OP, file);
  assemble_name (file, name);
  fprintf (file, "," HOST_WIDE_INT_PRINT_UNSIGNED ",%u\n",
	   size, align / BITS_PER_UNIT);
}
#endif

/* Utility function for targets to use in implementing
   ASM_OUTPUT_ALIGNED_BSS.  */

void
x86_output_aligned_bss (FILE *file, tree decl, const char *name,
		       	unsigned HOST_WIDE_INT size, unsigned align)
{
  if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
      && size > (unsigned int)ix86_section_threshold)
    switch_to_section (get_named_section (decl, ".lbss", 0));
  else
    switch_to_section (bss_section);
  ASM_OUTPUT_ALIGN (file, floor_log2 (align / BITS_PER_UNIT));
#ifdef ASM_DECLARE_OBJECT_NAME
  last_assemble_variable_decl = decl;
  ASM_DECLARE_OBJECT_NAME (file, name, decl);
#else
  /* Standard thing is just output label for the object.  */
  ASM_OUTPUT_LABEL (file, name);
#endif /* ASM_DECLARE_OBJECT_NAME */
  ASM_OUTPUT_SKIP (file, size ? size : 1);
}

/* Decide whether we must probe the stack before any space allocation
   on this target.  It's essentially TARGET_STACK_PROBE except when
   -fstack-check causes the stack to be already probed differently.  */

bool
ix86_target_stack_probe (void)
{
  /* Do not probe the stack twice if static stack checking is enabled.  */
  if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
    return false;

  return TARGET_STACK_PROBE;
}

/* Decide whether we can make a sibling call to a function.  DECL is the
   declaration of the function being targeted by the call and EXP is the
   CALL_EXPR representing the call.  */

static bool
ix86_function_ok_for_sibcall (tree decl, tree exp)
{
  tree type, decl_or_type;
  rtx a, b;
  bool bind_global = decl && !targetm.binds_local_p (decl);

  if (ix86_function_naked (current_function_decl))
    return false;

  /* Sibling call isn't OK if there are no caller-saved registers
     since all registers must be preserved before return.  */
  if (cfun->machine->no_caller_saved_registers)
    return false;

  /* If we are generating position-independent code, we cannot sibcall
     optimize direct calls to global functions, as the PLT requires
     %ebx be live. (Darwin does not have a PLT.)  */
  if (!TARGET_MACHO
      && !TARGET_64BIT
      && flag_pic
      && flag_plt
      && bind_global)
    return false;

  /* If we need to align the outgoing stack, then sibcalling would
     unalign the stack, which may break the called function.  */
  if (ix86_minimum_incoming_stack_boundary (true)
      < PREFERRED_STACK_BOUNDARY)
    return false;

  if (decl)
    {
      decl_or_type = decl;
      type = TREE_TYPE (decl);
    }
  else
    {
      /* We're looking at the CALL_EXPR, we need the type of the function.  */
      type = CALL_EXPR_FN (exp);		/* pointer expression */
      type = TREE_TYPE (type);			/* pointer type */
      type = TREE_TYPE (type);			/* function type */
      decl_or_type = type;
    }

  /* If outgoing reg parm stack space changes, we cannot do sibcall.  */
  if ((OUTGOING_REG_PARM_STACK_SPACE (type)
       != OUTGOING_REG_PARM_STACK_SPACE (TREE_TYPE (current_function_decl)))
      || (REG_PARM_STACK_SPACE (decl_or_type)
	  != REG_PARM_STACK_SPACE (current_function_decl)))
    {
      maybe_complain_about_tail_call (exp,
				      "inconsistent size of stack space"
				      " allocated for arguments which are"
				      " passed in registers");
      return false;
    }

  /* Check that the return value locations are the same.  Like
     if we are returning floats on the 80387 register stack, we cannot
     make a sibcall from a function that doesn't return a float to a
     function that does or, conversely, from a function that does return
     a float to a function that doesn't; the necessary stack adjustment
     would not be executed.  This is also the place we notice
     differences in the return value ABI.  Note that it is ok for one
     of the functions to have void return type as long as the return
     value of the other is passed in a register.  */
  a = ix86_function_value (TREE_TYPE (exp), decl_or_type, false);
  b = ix86_function_value (TREE_TYPE (DECL_RESULT (cfun->decl)),
			   cfun->decl, false);
  if (STACK_REG_P (a) || STACK_REG_P (b))
    {
      if (!rtx_equal_p (a, b))
	return false;
    }
  else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
    ;
  else if (!rtx_equal_p (a, b))
    return false;

  if (TARGET_64BIT)
    {
      /* The SYSV ABI has more call-clobbered registers;
	 disallow sibcalls from MS to SYSV.  */
      if (cfun->machine->call_abi == MS_ABI
	  && ix86_function_type_abi (type) == SYSV_ABI)
	return false;
    }
  else
    {
      /* If this call is indirect, we'll need to be able to use a
	 call-clobbered register for the address of the target function.
	 Make sure that all such registers are not used for passing
	 parameters.  Note that DLLIMPORT functions and call to global
	 function via GOT slot are indirect.  */
      if (!decl
	  || (bind_global && flag_pic && !flag_plt)
	  || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && DECL_DLLIMPORT_P (decl))
	  || flag_force_indirect_call)
	{
	  /* Check if regparm >= 3 since arg_reg_available is set to
	     false if regparm == 0.  If regparm is 1 or 2, there is
	     always a call-clobbered register available.

	     ??? The symbol indirect call doesn't need a call-clobbered
	     register.  But we don't know if this is a symbol indirect
	     call or not here.  */
	  if (ix86_function_regparm (type, decl) >= 3
	      && !cfun->machine->arg_reg_available)
	    return false;
	}
    }

  if (decl && ix86_use_pseudo_pic_reg ())
    {
      /* When PIC register is used, it must be restored after ifunc
	 function returns.  */
       cgraph_node *node = cgraph_node::get (decl);
       if (node && node->ifunc_resolver)
	 return false;
    }

  /* Disable sibcall if callee has indirect_return attribute and
     caller doesn't since callee will return to the caller's caller
     via an indirect jump.  */
  if (((flag_cf_protection & (CF_RETURN | CF_BRANCH))
       == (CF_RETURN | CF_BRANCH))
      && lookup_attribute ("indirect_return", TYPE_ATTRIBUTES (type))
      && !lookup_attribute ("indirect_return",
			    TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl))))
    return false;

  /* Otherwise okay.  That also includes certain types of indirect calls.  */
  return true;
}

/* This function determines from TYPE the calling-convention.  */

unsigned int
ix86_get_callcvt (const_tree type)
{
  unsigned int ret = 0;
  bool is_stdarg;
  tree attrs;

  if (TARGET_64BIT)
    return IX86_CALLCVT_CDECL;

  attrs = TYPE_ATTRIBUTES (type);
  if (attrs != NULL_TREE)
    {
      if (lookup_attribute ("cdecl", attrs))
	ret |= IX86_CALLCVT_CDECL;
      else if (lookup_attribute ("stdcall", attrs))
	ret |= IX86_CALLCVT_STDCALL;
      else if (lookup_attribute ("fastcall", attrs))
	ret |= IX86_CALLCVT_FASTCALL;
      else if (lookup_attribute ("thiscall", attrs))
	ret |= IX86_CALLCVT_THISCALL;

      /* Regparam isn't allowed for thiscall and fastcall.  */
      if ((ret & (IX86_CALLCVT_THISCALL | IX86_CALLCVT_FASTCALL)) == 0)
	{
	  if (lookup_attribute ("regparm", attrs))
	    ret |= IX86_CALLCVT_REGPARM;
	  if (lookup_attribute ("sseregparm", attrs))
	    ret |= IX86_CALLCVT_SSEREGPARM;
	}

      if (IX86_BASE_CALLCVT(ret) != 0)
	return ret;
    }

  is_stdarg = stdarg_p (type);
  if (TARGET_RTD && !is_stdarg)
    return IX86_CALLCVT_STDCALL | ret;

  if (ret != 0
      || is_stdarg
      || TREE_CODE (type) != METHOD_TYPE
      || ix86_function_type_abi (type) != MS_ABI)
    return IX86_CALLCVT_CDECL | ret;

  return IX86_CALLCVT_THISCALL;
}

/* Return 0 if the attributes for two types are incompatible, 1 if they
   are compatible, and 2 if they are nearly compatible (which causes a
   warning to be generated).  */

static int
ix86_comp_type_attributes (const_tree type1, const_tree type2)
{
  unsigned int ccvt1, ccvt2;

  if (TREE_CODE (type1) != FUNCTION_TYPE
      && TREE_CODE (type1) != METHOD_TYPE)
    return 1;

  ccvt1 = ix86_get_callcvt (type1);
  ccvt2 = ix86_get_callcvt (type2);
  if (ccvt1 != ccvt2)
    return 0;
  if (ix86_function_regparm (type1, NULL)
      != ix86_function_regparm (type2, NULL))
    return 0;

  return 1;
}

/* Return the regparm value for a function with the indicated TYPE and DECL.
   DECL may be NULL when calling function indirectly
   or considering a libcall.  */

static int
ix86_function_regparm (const_tree type, const_tree decl)
{
  tree attr;
  int regparm;
  unsigned int ccvt;

  if (TARGET_64BIT)
    return (ix86_function_type_abi (type) == SYSV_ABI
	    ? X86_64_REGPARM_MAX : X86_64_MS_REGPARM_MAX);
  ccvt = ix86_get_callcvt (type);
  regparm = ix86_regparm;

  if ((ccvt & IX86_CALLCVT_REGPARM) != 0)
    {
      attr = lookup_attribute ("regparm", TYPE_ATTRIBUTES (type));
      if (attr)
	{
	  regparm = TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr)));
	  return regparm;
	}
    }
  else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
    return 2;
  else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
    return 1;

  /* Use register calling convention for local functions when possible.  */
  if (decl
      && TREE_CODE (decl) == FUNCTION_DECL)
    {
      cgraph_node *target = cgraph_node::get (decl);
      if (target)
	target = target->function_symbol ();

      /* Caller and callee must agree on the calling convention, so
	 checking here just optimize means that with
	 __attribute__((optimize (...))) caller could use regparm convention
	 and callee not, or vice versa.  Instead look at whether the callee
	 is optimized or not.  */
      if (target && opt_for_fn (target->decl, optimize)
	  && !(profile_flag && !flag_fentry))
	{
	  if (target->local && target->can_change_signature)
	    {
	      int local_regparm, globals = 0, regno;

	      /* Make sure no regparm register is taken by a
		 fixed register variable.  */
	      for (local_regparm = 0; local_regparm < REGPARM_MAX;
		   local_regparm++)
		if (fixed_regs[local_regparm])
		  break;

	      /* We don't want to use regparm(3) for nested functions as
		 these use a static chain pointer in the third argument.  */
	      if (local_regparm == 3 && DECL_STATIC_CHAIN (target->decl))
		local_regparm = 2;

	      /* Save a register for the split stack.  */
	      if (flag_split_stack)
		{
		  if (local_regparm == 3)
		    local_regparm = 2;
		  else if (local_regparm == 2
			   && DECL_STATIC_CHAIN (target->decl))
		    local_regparm = 1;
		}

	      /* Each fixed register usage increases register pressure,
		 so less registers should be used for argument passing.
		 This functionality can be overriden by an explicit
		 regparm value.  */
	      for (regno = AX_REG; regno <= DI_REG; regno++)
		if (fixed_regs[regno])
		  globals++;

	      local_regparm
		= globals < local_regparm ? local_regparm - globals : 0;

	      if (local_regparm > regparm)
		regparm = local_regparm;
	    }
	}
    }

  return regparm;
}

/* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
   DFmode (2) arguments in SSE registers for a function with the
   indicated TYPE and DECL.  DECL may be NULL when calling function
   indirectly or considering a libcall.  Return -1 if any FP parameter
   should be rejected by error.  This is used in siutation we imply SSE
   calling convetion but the function is called from another function with
   SSE disabled. Otherwise return 0.  */

static int
ix86_function_sseregparm (const_tree type, const_tree decl, bool warn)
{
  gcc_assert (!TARGET_64BIT);

  /* Use SSE registers to pass SFmode and DFmode arguments if requested
     by the sseregparm attribute.  */
  if (TARGET_SSEREGPARM
      || (type && lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type))))
    {
      if (!TARGET_SSE)
	{
	  if (warn)
	    {
	      if (decl)
		error ("calling %qD with attribute sseregparm without "
		       "SSE/SSE2 enabled", decl);
	      else
		error ("calling %qT with attribute sseregparm without "
		       "SSE/SSE2 enabled", type);
	    }
	  return 0;
	}

      return 2;
    }

  if (!decl)
    return 0;

  cgraph_node *target = cgraph_node::get (decl);
  if (target)
    target = target->function_symbol ();

  /* For local functions, pass up to SSE_REGPARM_MAX SFmode
     (and DFmode for SSE2) arguments in SSE registers.  */
  if (target
      /* TARGET_SSE_MATH */
      && (target_opts_for_fn (target->decl)->x_ix86_fpmath & FPMATH_SSE)
      && opt_for_fn (target->decl, optimize)
      && !(profile_flag && !flag_fentry))
    {
      if (target->local && target->can_change_signature)
	{
	  /* Refuse to produce wrong code when local function with SSE enabled
	     is called from SSE disabled function.
	     FIXME: We need a way to detect these cases cross-ltrans partition
	     and avoid using SSE calling conventions on local functions called
	     from function with SSE disabled.  For now at least delay the
	     warning until we know we are going to produce wrong code.
	     See PR66047  */
	  if (!TARGET_SSE && warn)
	    return -1;
	  return TARGET_SSE2_P (target_opts_for_fn (target->decl)
				->x_ix86_isa_flags) ? 2 : 1;
	}
    }

  return 0;
}

/* Return true if EAX is live at the start of the function.  Used by
   ix86_expand_prologue to determine if we need special help before
   calling allocate_stack_worker.  */

static bool
ix86_eax_live_at_start_p (void)
{
  /* Cheat.  Don't bother working forward from ix86_function_regparm
     to the function type to whether an actual argument is located in
     eax.  Instead just look at cfg info, which is still close enough
     to correct at this point.  This gives false positives for broken
     functions that might use uninitialized data that happens to be
     allocated in eax, but who cares?  */
  return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 0);
}

static bool
ix86_keep_aggregate_return_pointer (tree fntype)
{
  tree attr;

  if (!TARGET_64BIT)
    {
      attr = lookup_attribute ("callee_pop_aggregate_return",
			       TYPE_ATTRIBUTES (fntype));
      if (attr)
	return (TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr))) == 0);

      /* For 32-bit MS-ABI the default is to keep aggregate
         return pointer.  */
      if (ix86_function_type_abi (fntype) == MS_ABI)
	return true;
    }
  return KEEP_AGGREGATE_RETURN_POINTER != 0;
}

/* Value is the number of bytes of arguments automatically
   popped when returning from a subroutine call.
   FUNDECL is the declaration node of the function (as a tree),
   FUNTYPE is the data type of the function (as a tree),
   or for a library call it is an identifier node for the subroutine name.
   SIZE is the number of bytes of arguments passed on the stack.

   On the 80386, the RTD insn may be used to pop them if the number
     of args is fixed, but if the number is variable then the caller
     must pop them all.  RTD can't be used for library calls now
     because the library is compiled with the Unix compiler.
   Use of RTD is a selectable option, since it is incompatible with
   standard Unix calling sequences.  If the option is not selected,
   the caller must always pop the args.

   The attribute stdcall is equivalent to RTD on a per module basis.  */

static poly_int64
ix86_return_pops_args (tree fundecl, tree funtype, poly_int64 size)
{
  unsigned int ccvt;

  /* None of the 64-bit ABIs pop arguments.  */
  if (TARGET_64BIT)
    return 0;

  ccvt = ix86_get_callcvt (funtype);

  if ((ccvt & (IX86_CALLCVT_STDCALL | IX86_CALLCVT_FASTCALL
	       | IX86_CALLCVT_THISCALL)) != 0
      && ! stdarg_p (funtype))
    return size;

  /* Lose any fake structure return argument if it is passed on the stack.  */
  if (aggregate_value_p (TREE_TYPE (funtype), fundecl)
      && !ix86_keep_aggregate_return_pointer (funtype))
    {
      int nregs = ix86_function_regparm (funtype, fundecl);
      if (nregs == 0)
	return GET_MODE_SIZE (Pmode);
    }

  return 0;
}

/* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook.  */

static bool
ix86_legitimate_combined_insn (rtx_insn *insn)
{
  int i;

  /* Check operand constraints in case hard registers were propagated
     into insn pattern.  This check prevents combine pass from
     generating insn patterns with invalid hard register operands.
     These invalid insns can eventually confuse reload to error out
     with a spill failure.  See also PRs 46829 and 46843.  */

  gcc_assert (INSN_CODE (insn) >= 0);

  extract_insn (insn);
  preprocess_constraints (insn);

  int n_operands = recog_data.n_operands;
  int n_alternatives = recog_data.n_alternatives;
  for (i = 0; i < n_operands; i++)
    {
      rtx op = recog_data.operand[i];
      machine_mode mode = GET_MODE (op);
      const operand_alternative *op_alt;
      int offset = 0;
      bool win;
      int j;

      /* A unary operator may be accepted by the predicate, but it
	 is irrelevant for matching constraints.  */
      if (UNARY_P (op))
	op = XEXP (op, 0);

      if (SUBREG_P (op))
	{
	  if (REG_P (SUBREG_REG (op))
	      && REGNO (SUBREG_REG (op)) < FIRST_PSEUDO_REGISTER)
	    offset = subreg_regno_offset (REGNO (SUBREG_REG (op)),
					  GET_MODE (SUBREG_REG (op)),
					  SUBREG_BYTE (op),
					  GET_MODE (op));
	  op = SUBREG_REG (op);
	}

      if (!(REG_P (op) && HARD_REGISTER_P (op)))
	continue;

      op_alt = recog_op_alt;

      /* Operand has no constraints, anything is OK.  */
      win = !n_alternatives;

      alternative_mask preferred = get_preferred_alternatives (insn);
      for (j = 0; j < n_alternatives; j++, op_alt += n_operands)
	{
	  if (!TEST_BIT (preferred, j))
	    continue;
	  if (op_alt[i].anything_ok
	      || (op_alt[i].matches != -1
		  && operands_match_p
		  (recog_data.operand[i],
		   recog_data.operand[op_alt[i].matches]))
	      || reg_fits_class_p (op, op_alt[i].cl, offset, mode))
	    {
	      win = true;
	      break;
	    }
	}

      if (!win)
	return false;
    }

  return true;
}

/* Implement the TARGET_ASAN_SHADOW_OFFSET hook.  */

static unsigned HOST_WIDE_INT
ix86_asan_shadow_offset (void)
{
  return SUBTARGET_SHADOW_OFFSET;
}

/* Argument support functions.  */

/* Return true when register may be used to pass function parameters.  */
bool
ix86_function_arg_regno_p (int regno)
{
  int i;
  enum calling_abi call_abi;
  const int *parm_regs;

  if (TARGET_SSE && SSE_REGNO_P (regno)
      && regno < FIRST_SSE_REG + SSE_REGPARM_MAX)
    return true;

   if (!TARGET_64BIT)
     return (regno < REGPARM_MAX
	     || (TARGET_MMX && MMX_REGNO_P (regno)
		 && regno < FIRST_MMX_REG + MMX_REGPARM_MAX));

  /* TODO: The function should depend on current function ABI but
     builtins.cc would need updating then. Therefore we use the
     default ABI.  */
  call_abi = ix86_cfun_abi ();

  /* RAX is used as hidden argument to va_arg functions.  */
  if (call_abi == SYSV_ABI && regno == AX_REG)
    return true;

  if (call_abi == MS_ABI)
    parm_regs = x86_64_ms_abi_int_parameter_registers;
  else
    parm_regs = x86_64_int_parameter_registers;

  for (i = 0; i < (call_abi == MS_ABI
		   ? X86_64_MS_REGPARM_MAX : X86_64_REGPARM_MAX); i++)
    if (regno == parm_regs[i])
      return true;
  return false;
}

/* Return if we do not know how to pass ARG solely in registers.  */

static bool
ix86_must_pass_in_stack (const function_arg_info &arg)
{
  if (must_pass_in_stack_var_size_or_pad (arg))
    return true;

  /* For 32-bit, we want TImode aggregates to go on the stack.  But watch out!
     The layout_type routine is crafty and tries to trick us into passing
     currently unsupported vector types on the stack by using TImode.  */
  return (!TARGET_64BIT && arg.mode == TImode
	  && arg.type && TREE_CODE (arg.type) != VECTOR_TYPE);
}

/* It returns the size, in bytes, of the area reserved for arguments passed
   in registers for the function represented by fndecl dependent to the used
   abi format.  */
int
ix86_reg_parm_stack_space (const_tree fndecl)
{
  enum calling_abi call_abi = SYSV_ABI;
  if (fndecl != NULL_TREE && TREE_CODE (fndecl) == FUNCTION_DECL)
    call_abi = ix86_function_abi (fndecl);
  else
    call_abi = ix86_function_type_abi (fndecl);
  if (TARGET_64BIT && call_abi == MS_ABI)
    return 32;
  return 0;
}

/* We add this as a workaround in order to use libc_has_function
   hook in i386.md.  */
bool
ix86_libc_has_function (enum function_class fn_class)
{
  return targetm.libc_has_function (fn_class, NULL_TREE);
}

/* Returns value SYSV_ABI, MS_ABI dependent on fntype,
   specifying the call abi used.  */
enum calling_abi
ix86_function_type_abi (const_tree fntype)
{
  enum calling_abi abi = ix86_abi;

  if (fntype == NULL_TREE || TYPE_ATTRIBUTES (fntype) == NULL_TREE)
    return abi;

  if (abi == SYSV_ABI
      && lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (fntype)))
    {
      static int warned;
      if (TARGET_X32 && !warned)
	{
	  error ("X32 does not support %<ms_abi%> attribute");
	  warned = 1;
	}

      abi = MS_ABI;
    }
  else if (abi == MS_ABI
	   && lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (fntype)))
    abi = SYSV_ABI;

  return abi;
}

enum calling_abi
ix86_function_abi (const_tree fndecl)
{
  return fndecl ? ix86_function_type_abi (TREE_TYPE (fndecl)) : ix86_abi;
}

/* Returns value SYSV_ABI, MS_ABI dependent on cfun,
   specifying the call abi used.  */
enum calling_abi
ix86_cfun_abi (void)
{
  return cfun ? cfun->machine->call_abi : ix86_abi;
}

bool
ix86_function_ms_hook_prologue (const_tree fn)
{
  if (fn && lookup_attribute ("ms_hook_prologue", DECL_ATTRIBUTES (fn)))
    {
      if (decl_function_context (fn) != NULL_TREE)
	error_at (DECL_SOURCE_LOCATION (fn),
		  "%<ms_hook_prologue%> attribute is not compatible "
		  "with nested function");
      else
        return true;
    }
  return false;
}

bool
ix86_function_naked (const_tree fn)
{
  if (fn && lookup_attribute ("naked", DECL_ATTRIBUTES (fn)))
    return true;

  return false;
}

/* Write the extra assembler code needed to declare a function properly.  */

void
ix86_asm_output_function_label (FILE *out_file, const char *fname,
				tree decl)
{
  bool is_ms_hook = ix86_function_ms_hook_prologue (decl);

  if (cfun)
    cfun->machine->function_label_emitted = true;

  if (is_ms_hook)
    {
      int i, filler_count = (TARGET_64BIT ? 32 : 16);
      unsigned int filler_cc = 0xcccccccc;

      for (i = 0; i < filler_count; i += 4)
	fprintf (out_file, ASM_LONG " %#x\n", filler_cc);
    }

#ifdef SUBTARGET_ASM_UNWIND_INIT
  SUBTARGET_ASM_UNWIND_INIT (out_file);
#endif

  ASM_OUTPUT_LABEL (out_file, fname);

  /* Output magic byte marker, if hot-patch attribute is set.  */
  if (is_ms_hook)
    {
      if (TARGET_64BIT)
	{
	  /* leaq [%rsp + 0], %rsp  */
	  fputs (ASM_BYTE "0x48, 0x8d, 0xa4, 0x24, 0x00, 0x00, 0x00, 0x00\n",
		 out_file);
	}
      else
	{
          /* movl.s %edi, %edi
	     push   %ebp
	     movl.s %esp, %ebp */
	  fputs (ASM_BYTE "0x8b, 0xff, 0x55, 0x8b, 0xec\n", out_file);
	}
    }
}

/* Implementation of call abi switching target hook. Specific to FNDECL
   the specific call register sets are set.  See also
   ix86_conditional_register_usage for more details.  */
void
ix86_call_abi_override (const_tree fndecl)
{
  cfun->machine->call_abi = ix86_function_abi (fndecl);
}

/* Return 1 if pseudo register should be created and used to hold
   GOT address for PIC code.  */
bool
ix86_use_pseudo_pic_reg (void)
{
  if ((TARGET_64BIT
       && (ix86_cmodel == CM_SMALL_PIC
	   || TARGET_PECOFF))
      || !flag_pic)
    return false;
  return true;
}

/* Initialize large model PIC register.  */

static void
ix86_init_large_pic_reg (unsigned int tmp_regno)
{
  rtx_code_label *label;
  rtx tmp_reg;

  gcc_assert (Pmode == DImode);
  label = gen_label_rtx ();
  emit_label (label);
  LABEL_PRESERVE_P (label) = 1;
  tmp_reg = gen_rtx_REG (Pmode, tmp_regno);
  gcc_assert (REGNO (pic_offset_table_rtx) != tmp_regno);
  emit_insn (gen_set_rip_rex64 (pic_offset_table_rtx,
				label));
  emit_insn (gen_set_got_offset_rex64 (tmp_reg, label));
  emit_insn (gen_add2_insn (pic_offset_table_rtx, tmp_reg));
  const char *name = LABEL_NAME (label);
  PUT_CODE (label, NOTE);
  NOTE_KIND (label) = NOTE_INSN_DELETED_LABEL;
  NOTE_DELETED_LABEL_NAME (label) = name;
}

/* Create and initialize PIC register if required.  */
static void
ix86_init_pic_reg (void)
{
  edge entry_edge;
  rtx_insn *seq;

  if (!ix86_use_pseudo_pic_reg ())
    return;

  start_sequence ();

  if (TARGET_64BIT)
    {
      if (ix86_cmodel == CM_LARGE_PIC)
	ix86_init_large_pic_reg (R11_REG);
      else
	emit_insn (gen_set_got_rex64 (pic_offset_table_rtx));
    }
  else
    {
      /*  If there is future mcount call in the function it is more profitable
	  to emit SET_GOT into ABI defined REAL_PIC_OFFSET_TABLE_REGNUM.  */
      rtx reg = crtl->profile
		? gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM)
		: pic_offset_table_rtx;
      rtx_insn *insn = emit_insn (gen_set_got (reg));
      RTX_FRAME_RELATED_P (insn) = 1;
      if (crtl->profile)
        emit_move_insn (pic_offset_table_rtx, reg);
      add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
    }

  seq = get_insns ();
  end_sequence ();

  entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
  insert_insn_on_edge (seq, entry_edge);
  commit_one_edge_insertion (entry_edge);
}

/* Initialize a variable CUM of type CUMULATIVE_ARGS
   for a call to a function whose data type is FNTYPE.
   For a library call, FNTYPE is 0.  */

void
init_cumulative_args (CUMULATIVE_ARGS *cum,  /* Argument info to initialize */
		      tree fntype,	/* tree ptr for function decl */
		      rtx libname,	/* SYMBOL_REF of library name or 0 */
		      tree fndecl,
		      int caller)
{
  struct cgraph_node *local_info_node = NULL;
  struct cgraph_node *target = NULL;

  /* Set silent_p to false to raise an error for invalid calls when
     expanding function body.  */
  cfun->machine->silent_p = false;

  memset (cum, 0, sizeof (*cum));

  if (fndecl)
    {
      target = cgraph_node::get (fndecl);
      if (target)
	{
	  target = target->function_symbol ();
	  local_info_node = cgraph_node::local_info_node (target->decl);
	  cum->call_abi = ix86_function_abi (target->decl);
	}
      else
	cum->call_abi = ix86_function_abi (fndecl);
    }
  else
    cum->call_abi = ix86_function_type_abi (fntype);

  cum->caller = caller;

  /* Set up the number of registers to use for passing arguments.  */
  cum->nregs = ix86_regparm;
  if (TARGET_64BIT)
    {
      cum->nregs = (cum->call_abi == SYSV_ABI
                   ? X86_64_REGPARM_MAX
                   : X86_64_MS_REGPARM_MAX);
    }
  if (TARGET_SSE)
    {
      cum->sse_nregs = SSE_REGPARM_MAX;
      if (TARGET_64BIT)
        {
          cum->sse_nregs = (cum->call_abi == SYSV_ABI
                           ? X86_64_SSE_REGPARM_MAX
                           : X86_64_MS_SSE_REGPARM_MAX);
        }
    }
  if (TARGET_MMX)
    cum->mmx_nregs = MMX_REGPARM_MAX;
  cum->warn_avx512f = true;
  cum->warn_avx = true;
  cum->warn_sse = true;
  cum->warn_mmx = true;

  /* Because type might mismatch in between caller and callee, we need to
     use actual type of function for local calls.
     FIXME: cgraph_analyze can be told to actually record if function uses
     va_start so for local functions maybe_vaarg can be made aggressive
     helping K&R code.
     FIXME: once typesytem is fixed, we won't need this code anymore.  */
  if (local_info_node && local_info_node->local
      && local_info_node->can_change_signature)
    fntype = TREE_TYPE (target->decl);
  cum->stdarg = stdarg_p (fntype);
  cum->maybe_vaarg = (fntype
		      ? (!prototype_p (fntype) || stdarg_p (fntype))
		      : !libname);

  cum->decl = fndecl;

  cum->warn_empty = !warn_abi || cum->stdarg;
  if (!cum->warn_empty && fntype)
    {
      function_args_iterator iter;
      tree argtype;
      bool seen_empty_type = false;
      FOREACH_FUNCTION_ARGS (fntype, argtype, iter)
	{
	  if (argtype == error_mark_node || VOID_TYPE_P (argtype))
	    break;
	  if (TYPE_EMPTY_P (argtype))
	    seen_empty_type = true;
	  else if (seen_empty_type)
	    {
	      cum->warn_empty = true;
	      break;
	    }
	}
    }

  if (!TARGET_64BIT)
    {
      /* If there are variable arguments, then we won't pass anything
         in registers in 32-bit mode. */
      if (stdarg_p (fntype))
	{
	  cum->nregs = 0;
	  /* Since in 32-bit, variable arguments are always passed on
	     stack, there is scratch register available for indirect
	     sibcall.  */
	  cfun->machine->arg_reg_available = true;
	  cum->sse_nregs = 0;
	  cum->mmx_nregs = 0;
	  cum->warn_avx512f = false;
	  cum->warn_avx = false;
	  cum->warn_sse = false;
	  cum->warn_mmx = false;
	  return;
	}

      /* Use ecx and edx registers if function has fastcall attribute,
	 else look for regparm information.  */
      if (fntype)
	{
	  unsigned int ccvt = ix86_get_callcvt (fntype);
	  if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
	    {
	      cum->nregs = 1;
	      cum->fastcall = 1; /* Same first register as in fastcall.  */
	    }
	  else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
	    {
	      cum->nregs = 2;
	      cum->fastcall = 1;
	    }
	  else
	    cum->nregs = ix86_function_regparm (fntype, fndecl);
	}

      /* Set up the number of SSE registers used for passing SFmode
	 and DFmode arguments.  Warn for mismatching ABI.  */
      cum->float_in_sse = ix86_function_sseregparm (fntype, fndecl, true);
    }

  cfun->machine->arg_reg_available = (cum->nregs > 0);
}

/* Return the "natural" mode for TYPE.  In most cases, this is just TYPE_MODE.
   But in the case of vector types, it is some vector mode.

   When we have only some of our vector isa extensions enabled, then there
   are some modes for which vector_mode_supported_p is false.  For these
   modes, the generic vector support in gcc will choose some non-vector mode
   in order to implement the type.  By computing the natural mode, we'll
   select the proper ABI location for the operand and not depend on whatever
   the middle-end decides to do with these vector types.

   The midde-end can't deal with the vector types > 16 bytes.  In this
   case, we return the original mode and warn ABI change if CUM isn't
   NULL. 

   If INT_RETURN is true, warn ABI change if the vector mode isn't
   available for function return value.  */

static machine_mode
type_natural_mode (const_tree type, const CUMULATIVE_ARGS *cum,
		   bool in_return)
{
  machine_mode mode = TYPE_MODE (type);

  if (VECTOR_TYPE_P (type) && !VECTOR_MODE_P (mode))
    {
      HOST_WIDE_INT size = int_size_in_bytes (type);
      if ((size == 8 || size == 16 || size == 32 || size == 64)
	  /* ??? Generic code allows us to create width 1 vectors.  Ignore.  */
	  && TYPE_VECTOR_SUBPARTS (type) > 1)
	{
	  machine_mode innermode = TYPE_MODE (TREE_TYPE (type));

	  /* There are no XFmode vector modes ...  */
	  if (innermode == XFmode)
	    return mode;

	  /* ... and no decimal float vector modes.  */
	  if (DECIMAL_FLOAT_MODE_P (innermode))
	    return mode;

	  if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (type)))
	    mode = MIN_MODE_VECTOR_FLOAT;
	  else
	    mode = MIN_MODE_VECTOR_INT;

	  /* Get the mode which has this inner mode and number of units.  */
	  FOR_EACH_MODE_FROM (mode, mode)
	    if (GET_MODE_NUNITS (mode) == TYPE_VECTOR_SUBPARTS (type)
		&& GET_MODE_INNER (mode) == innermode)
	      {
		if (size == 64 && !TARGET_AVX512F && !TARGET_IAMCU)
		  {
		    static bool warnedavx512f;
		    static bool warnedavx512f_ret;

		    if (cum && cum->warn_avx512f && !warnedavx512f)
		      {
			if (warning (OPT_Wpsabi, "AVX512F vector argument "
				     "without AVX512F enabled changes the ABI"))
			  warnedavx512f = true;
		      }
		    else if (in_return && !warnedavx512f_ret)
		      {
			if (warning (OPT_Wpsabi, "AVX512F vector return "
				     "without AVX512F enabled changes the ABI"))
			  warnedavx512f_ret = true;
		      }

		    return TYPE_MODE (type);
		  }
		else if (size == 32 && !TARGET_AVX && !TARGET_IAMCU)
		  {
		    static bool warnedavx;
		    static bool warnedavx_ret;

		    if (cum && cum->warn_avx && !warnedavx)
		      {
			if (warning (OPT_Wpsabi, "AVX vector argument "
				     "without AVX enabled changes the ABI"))
			  warnedavx = true;
		      }
		    else if (in_return && !warnedavx_ret)
		      {
			if (warning (OPT_Wpsabi, "AVX vector return "
				     "without AVX enabled changes the ABI"))
			  warnedavx_ret = true;
		      }

		    return TYPE_MODE (type);
		  }
		else if (((size == 8 && TARGET_64BIT) || size == 16)
			 && !TARGET_SSE
			 && !TARGET_IAMCU)
		  {
		    static bool warnedsse;
		    static bool warnedsse_ret;

		    if (cum && cum->warn_sse && !warnedsse)
		      {
			if (warning (OPT_Wpsabi, "SSE vector argument "
				     "without SSE enabled changes the ABI"))
			  warnedsse = true;
		      }
		    else if (!TARGET_64BIT && in_return && !warnedsse_ret)
		      {
			if (warning (OPT_Wpsabi, "SSE vector return "
				     "without SSE enabled changes the ABI"))
			  warnedsse_ret = true;
		      }
		  }
		else if ((size == 8 && !TARGET_64BIT)
			 && (!cfun
			     || cfun->machine->func_type == TYPE_NORMAL)
			 && !TARGET_MMX
			 && !TARGET_IAMCU)
		  {
		    static bool warnedmmx;
		    static bool warnedmmx_ret;

		    if (cum && cum->warn_mmx && !warnedmmx)
		      {
			if (warning (OPT_Wpsabi, "MMX vector argument "
				     "without MMX enabled changes the ABI"))
			  warnedmmx = true;
		      }
		    else if (in_return && !warnedmmx_ret)
		      {
			if (warning (OPT_Wpsabi, "MMX vector return "
				     "without MMX enabled changes the ABI"))
			  warnedmmx_ret = true;
		      }
		  }
		return mode;
	      }

	  gcc_unreachable ();
	}
    }

  return mode;
}

/* We want to pass a value in REGNO whose "natural" mode is MODE.  However,
   this may not agree with the mode that the type system has chosen for the
   register, which is ORIG_MODE.  If ORIG_MODE is not BLKmode, then we can
   go ahead and use it.  Otherwise we have to build a PARALLEL instead.  */

static rtx
gen_reg_or_parallel (machine_mode mode, machine_mode orig_mode,
		     unsigned int regno)
{
  rtx tmp;

  if (orig_mode != BLKmode)
    tmp = gen_rtx_REG (orig_mode, regno);
  else
    {
      tmp = gen_rtx_REG (mode, regno);
      tmp = gen_rtx_EXPR_LIST (VOIDmode, tmp, const0_rtx);
      tmp = gen_rtx_PARALLEL (orig_mode, gen_rtvec (1, tmp));
    }

  return tmp;
}

/* x86-64 register passing implementation.  See x86-64 ABI for details.  Goal
   of this code is to classify each 8bytes of incoming argument by the register
   class and assign registers accordingly.  */

/* Return the union class of CLASS1 and CLASS2.
   See the x86-64 PS ABI for details.  */

static enum x86_64_reg_class
merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
{
  /* Rule #1: If both classes are equal, this is the resulting class.  */
  if (class1 == class2)
    return class1;

  /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
     the other class.  */
  if (class1 == X86_64_NO_CLASS)
    return class2;
  if (class2 == X86_64_NO_CLASS)
    return class1;

  /* Rule #3: If one of the classes is MEMORY, the result is MEMORY.  */
  if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
    return X86_64_MEMORY_CLASS;

  /* Rule #4: If one of the classes is INTEGER, the result is INTEGER.  */
  if ((class1 == X86_64_INTEGERSI_CLASS
       && (class2 == X86_64_SSESF_CLASS || class2 == X86_64_SSEHF_CLASS))
      || (class2 == X86_64_INTEGERSI_CLASS
	  && (class1 == X86_64_SSESF_CLASS || class1 == X86_64_SSEHF_CLASS)))
    return X86_64_INTEGERSI_CLASS;
  if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
      || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
    return X86_64_INTEGER_CLASS;

  /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
     MEMORY is used.  */
  if (class1 == X86_64_X87_CLASS
      || class1 == X86_64_X87UP_CLASS
      || class1 == X86_64_COMPLEX_X87_CLASS
      || class2 == X86_64_X87_CLASS
      || class2 == X86_64_X87UP_CLASS
      || class2 == X86_64_COMPLEX_X87_CLASS)
    return X86_64_MEMORY_CLASS;

  /* Rule #6: Otherwise class SSE is used.  */
  return X86_64_SSE_CLASS;
}

/* Classify the argument of type TYPE and mode MODE.
   CLASSES will be filled by the register class used to pass each word
   of the operand.  The number of words is returned.  In case the parameter
   should be passed in memory, 0 is returned. As a special case for zero
   sized containers, classes[0] will be NO_CLASS and 1 is returned.

   BIT_OFFSET is used internally for handling records and specifies offset
   of the offset in bits modulo 512 to avoid overflow cases.

   See the x86-64 PS ABI for details.
*/

static int
classify_argument (machine_mode mode, const_tree type,
		   enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset,
		   int &zero_width_bitfields)
{
  HOST_WIDE_INT bytes
    = mode == BLKmode ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
  int words = CEIL (bytes + (bit_offset % 64) / 8, UNITS_PER_WORD);

  /* Variable sized entities are always passed/returned in memory.  */
  if (bytes < 0)
    return 0;

  if (mode != VOIDmode)
    {
      /* The value of "named" doesn't matter.  */
      function_arg_info arg (const_cast<tree> (type), mode, /*named=*/true);
      if (targetm.calls.must_pass_in_stack (arg))
	return 0;
    }

  if (type && (AGGREGATE_TYPE_P (type)
	       || (TREE_CODE (type) == BITINT_TYPE && words > 1)))
    {
      int i;
      tree field;
      enum x86_64_reg_class subclasses[MAX_CLASSES];

      /* On x86-64 we pass structures larger than 64 bytes on the stack.  */
      if (bytes > 64)
	return 0;

      for (i = 0; i < words; i++)
	classes[i] = X86_64_NO_CLASS;

      /* Zero sized arrays or structures are NO_CLASS.  We return 0 to
	 signalize memory class, so handle it as special case.  */
      if (!words)
	{
	  classes[0] = X86_64_NO_CLASS;
	  return 1;
	}

      /* Classify each field of record and merge classes.  */
      switch (TREE_CODE (type))
	{
	case RECORD_TYPE:
	  /* And now merge the fields of structure.  */
	  for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
	    {
	      if (TREE_CODE (field) == FIELD_DECL)
		{
		  int num;

		  if (TREE_TYPE (field) == error_mark_node)
		    continue;

		  /* Bitfields are always classified as integer.  Handle them
		     early, since later code would consider them to be
		     misaligned integers.  */
		  if (DECL_BIT_FIELD (field))
		    {
		      if (integer_zerop (DECL_SIZE (field)))
			{
			  if (DECL_FIELD_CXX_ZERO_WIDTH_BIT_FIELD (field))
			    continue;
			  if (zero_width_bitfields != 2)
			    {
			      zero_width_bitfields = 1;
			      continue;
			    }
			}
		      for (i = (int_bit_position (field)
				+ (bit_offset % 64)) / 8 / 8;
			   i < ((int_bit_position (field) + (bit_offset % 64))
			        + tree_to_shwi (DECL_SIZE (field))
				+ 63) / 8 / 8; i++)
			classes[i]
			  = merge_classes (X86_64_INTEGER_CLASS, classes[i]);
		    }
		  else
		    {
		      int pos;

		      type = TREE_TYPE (field);

		      /* Flexible array member is ignored.  */
		      if (TYPE_MODE (type) == BLKmode
			  && TREE_CODE (type) == ARRAY_TYPE
			  && TYPE_SIZE (type) == NULL_TREE
			  && TYPE_DOMAIN (type) != NULL_TREE
			  && (TYPE_MAX_VALUE (TYPE_DOMAIN (type))
			      == NULL_TREE))
			{
			  static bool warned;

			  if (!warned && warn_psabi)
			    {
			      warned = true;
			      inform (input_location,
				      "the ABI of passing struct with"
				      " a flexible array member has"
				      " changed in GCC 4.4");
			    }
			  continue;
			}
		      num = classify_argument (TYPE_MODE (type), type,
					       subclasses,
					       (int_bit_position (field)
						+ bit_offset) % 512,
					       zero_width_bitfields);
		      if (!num)
			return 0;
		      pos = (int_bit_position (field)
			     + (bit_offset % 64)) / 8 / 8;
		      for (i = 0; i < num && (i + pos) < words; i++)
			classes[i + pos]
			  = merge_classes (subclasses[i], classes[i + pos]);
		    }
		}
	    }
	  break;

	case ARRAY_TYPE:
	  /* Arrays are handled as small records.  */
	  {
	    int num;
	    num = classify_argument (TYPE_MODE (TREE_TYPE (type)),
				     TREE_TYPE (type), subclasses, bit_offset,
				     zero_width_bitfields);
	    if (!num)
	      return 0;

	    /* The partial classes are now full classes.  */
	    if (subclasses[0] == X86_64_SSESF_CLASS && bytes != 4)
	      subclasses[0] = X86_64_SSE_CLASS;
	    if (subclasses[0] == X86_64_SSEHF_CLASS && bytes != 2)
	      subclasses[0] = X86_64_SSE_CLASS;
	    if (subclasses[0] == X86_64_INTEGERSI_CLASS
		&& !((bit_offset % 64) == 0 && bytes == 4))
	      subclasses[0] = X86_64_INTEGER_CLASS;

	    for (i = 0; i < words; i++)
	      classes[i] = subclasses[i % num];

	    break;
	  }
	case UNION_TYPE:
	case QUAL_UNION_TYPE:
	  /* Unions are similar to RECORD_TYPE but offset is always 0.
	     */
	  for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
	    {
	      if (TREE_CODE (field) == FIELD_DECL)
		{
		  int num;

		  if (TREE_TYPE (field) == error_mark_node)
		    continue;

		  num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
					   TREE_TYPE (field), subclasses,
					   bit_offset, zero_width_bitfields);
		  if (!num)
		    return 0;
		  for (i = 0; i < num && i < words; i++)
		    classes[i] = merge_classes (subclasses[i], classes[i]);
		}
	    }
	  break;

	case BITINT_TYPE:
	  /* _BitInt(N) for N > 64 is passed as structure containing
	     (N + 63) / 64 64-bit elements.  */
	  if (words > 2)
	    return 0;
	  classes[0] = classes[1] = X86_64_INTEGER_CLASS;
	  return 2;

	default:
	  gcc_unreachable ();
	}

      if (words > 2)
	{
	  /* When size > 16 bytes, if the first one isn't
	     X86_64_SSE_CLASS or any other ones aren't
	     X86_64_SSEUP_CLASS, everything should be passed in
	     memory.  */
	  if (classes[0] != X86_64_SSE_CLASS)
	    return 0;

	  for (i = 1; i < words; i++)
	    if (classes[i] != X86_64_SSEUP_CLASS)
	      return 0;
	}

      /* Final merger cleanup.  */
      for (i = 0; i < words; i++)
	{
	  /* If one class is MEMORY, everything should be passed in
	     memory.  */
	  if (classes[i] == X86_64_MEMORY_CLASS)
	    return 0;

	  /* The X86_64_SSEUP_CLASS should be always preceded by
	     X86_64_SSE_CLASS or X86_64_SSEUP_CLASS.  */
	  if (classes[i] == X86_64_SSEUP_CLASS
	      && classes[i - 1] != X86_64_SSE_CLASS
	      && classes[i - 1] != X86_64_SSEUP_CLASS)
	    {
	      /* The first one should never be X86_64_SSEUP_CLASS.  */
	      gcc_assert (i != 0);
	      classes[i] = X86_64_SSE_CLASS;
	    }

	  /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
	     everything should be passed in memory.  */
	  if (classes[i] == X86_64_X87UP_CLASS
	      && (classes[i - 1] != X86_64_X87_CLASS))
	    {
	      static bool warned;

	      /* The first one should never be X86_64_X87UP_CLASS.  */
	      gcc_assert (i != 0);
	      if (!warned && warn_psabi)
		{
		  warned = true;
		  inform (input_location,
			  "the ABI of passing union with %<long double%>"
			  " has changed in GCC 4.4");
		}
	      return 0;
	    }
	}
      return words;
    }

  /* Compute alignment needed.  We align all types to natural boundaries with
     exception of XFmode that is aligned to 64bits.  */
  if (mode != VOIDmode && mode != BLKmode)
    {
      int mode_alignment = GET_MODE_BITSIZE (mode);

      if (mode == XFmode)
	mode_alignment = 128;
      else if (mode == XCmode)
	mode_alignment = 256;
      if (COMPLEX_MODE_P (mode))
	mode_alignment /= 2;
      /* Misaligned fields are always returned in memory.  */
      if (bit_offset % mode_alignment)
	return 0;
    }

  /* for V1xx modes, just use the base mode */
  if (VECTOR_MODE_P (mode) && mode != V1DImode && mode != V1TImode
      && GET_MODE_UNIT_SIZE (mode) == bytes)
    mode = GET_MODE_INNER (mode);

  /* Classification of atomic types.  */
  switch (mode)
    {
    case E_SDmode:
    case E_DDmode:
      classes[0] = X86_64_SSE_CLASS;
      return 1;
    case E_TDmode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      return 2;
    case E_DImode:
    case E_SImode:
    case E_HImode:
    case E_QImode:
    case E_CSImode:
    case E_CHImode:
    case E_CQImode:
      {
	int size = bit_offset + (int) GET_MODE_BITSIZE (mode);

	/* Analyze last 128 bits only.  */
	size = (size - 1) & 0x7f;

	if (size < 32)
	  {
	    classes[0] = X86_64_INTEGERSI_CLASS;
	    return 1;
	  }
	else if (size < 64)
	  {
	    classes[0] = X86_64_INTEGER_CLASS;
	    return 1;
	  }
	else if (size < 64+32)
	  {
	    classes[0] = X86_64_INTEGER_CLASS;
	    classes[1] = X86_64_INTEGERSI_CLASS;
	    return 2;
	  }
	else if (size < 64+64)
	  {
	    classes[0] = classes[1] = X86_64_INTEGER_CLASS;
	    return 2;
	  }
	else
	  gcc_unreachable ();
      }
    case E_CDImode:
    case E_TImode:
      classes[0] = classes[1] = X86_64_INTEGER_CLASS;
      return 2;
    case E_COImode:
    case E_OImode:
      /* OImode shouldn't be used directly.  */
      gcc_unreachable ();
    case E_CTImode:
      return 0;
    case E_HFmode:
    case E_BFmode:
      if (!(bit_offset % 64))
	classes[0] = X86_64_SSEHF_CLASS;
      else
	classes[0] = X86_64_SSE_CLASS;
      return 1;
    case E_SFmode:
      if (!(bit_offset % 64))
	classes[0] = X86_64_SSESF_CLASS;
      else
	classes[0] = X86_64_SSE_CLASS;
      return 1;
    case E_DFmode:
      classes[0] = X86_64_SSEDF_CLASS;
      return 1;
    case E_XFmode:
      classes[0] = X86_64_X87_CLASS;
      classes[1] = X86_64_X87UP_CLASS;
      return 2;
    case E_TFmode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      return 2;
    case E_HCmode:
    case E_BCmode:
      classes[0] = X86_64_SSE_CLASS;
      if (!(bit_offset % 64))
	return 1;
      else
	{
	  classes[1] = X86_64_SSEHF_CLASS;
	  return 2;
	}
    case E_SCmode:
      classes[0] = X86_64_SSE_CLASS;
      if (!(bit_offset % 64))
	return 1;
      else
	{
	  static bool warned;

	  if (!warned && warn_psabi)
	    {
	      warned = true;
	      inform (input_location,
		      "the ABI of passing structure with %<complex float%>"
		      " member has changed in GCC 4.4");
	    }
	  classes[1] = X86_64_SSESF_CLASS;
	  return 2;
	}
    case E_DCmode:
      classes[0] = X86_64_SSEDF_CLASS;
      classes[1] = X86_64_SSEDF_CLASS;
      return 2;
    case E_XCmode:
      classes[0] = X86_64_COMPLEX_X87_CLASS;
      return 1;
    case E_TCmode:
      /* This modes is larger than 16 bytes.  */
      return 0;
    case E_V8SFmode:
    case E_V8SImode:
    case E_V32QImode:
    case E_V16HFmode:
    case E_V16BFmode:
    case E_V16HImode:
    case E_V4DFmode:
    case E_V4DImode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      classes[2] = X86_64_SSEUP_CLASS;
      classes[3] = X86_64_SSEUP_CLASS;
      return 4;
    case E_V8DFmode:
    case E_V16SFmode:
    case E_V32HFmode:
    case E_V32BFmode:
    case E_V8DImode:
    case E_V16SImode:
    case E_V32HImode:
    case E_V64QImode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      classes[2] = X86_64_SSEUP_CLASS;
      classes[3] = X86_64_SSEUP_CLASS;
      classes[4] = X86_64_SSEUP_CLASS;
      classes[5] = X86_64_SSEUP_CLASS;
      classes[6] = X86_64_SSEUP_CLASS;
      classes[7] = X86_64_SSEUP_CLASS;
      return 8;
    case E_V4SFmode:
    case E_V4SImode:
    case E_V16QImode:
    case E_V8HImode:
    case E_V8HFmode:
    case E_V8BFmode:
    case E_V2DFmode:
    case E_V2DImode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      return 2;
    case E_V1TImode:
    case E_V1DImode:
    case E_V2SFmode:
    case E_V2SImode:
    case E_V4HImode:
    case E_V4HFmode:
    case E_V4BFmode:
    case E_V2HFmode:
    case E_V2BFmode:
    case E_V8QImode:
      classes[0] = X86_64_SSE_CLASS;
      return 1;
    case E_BLKmode:
    case E_VOIDmode:
      return 0;
    default:
      gcc_assert (VECTOR_MODE_P (mode));

      if (bytes > 16)
	return 0;

      gcc_assert (GET_MODE_CLASS (GET_MODE_INNER (mode)) == MODE_INT);

      if (bit_offset + GET_MODE_BITSIZE (mode) <= 32)
	classes[0] = X86_64_INTEGERSI_CLASS;
      else
	classes[0] = X86_64_INTEGER_CLASS;
      classes[1] = X86_64_INTEGER_CLASS;
      return 1 + (bytes > 8);
    }
}

/* Wrapper around classify_argument with the extra zero_width_bitfields
   argument, to diagnose GCC 12.1 ABI differences for C.  */

static int
classify_argument (machine_mode mode, const_tree type,
		   enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset)
{
  int zero_width_bitfields = 0;
  static bool warned = false;
  int n = classify_argument (mode, type, classes, bit_offset,
			     zero_width_bitfields);
  if (!zero_width_bitfields || warned || !warn_psabi)
    return n;
  enum x86_64_reg_class alt_classes[MAX_CLASSES];
  zero_width_bitfields = 2;
  if (classify_argument (mode, type, alt_classes, bit_offset,
			 zero_width_bitfields) != n)
    zero_width_bitfields = 3;
  else
    for (int i = 0; i < n; i++)
      if (classes[i] != alt_classes[i])
	{
	  zero_width_bitfields = 3;
	  break;
	}
  if (zero_width_bitfields == 3)
    {
      warned = true;
      const char *url
	= CHANGES_ROOT_URL "gcc-12/changes.html#zero_width_bitfields";

      inform (input_location,
	      "the ABI of passing C structures with zero-width bit-fields"
	      " has changed in GCC %{12.1%}", url);
    }
  return n;
}

/* Examine the argument and return set number of register required in each
   class.  Return true iff parameter should be passed in memory.  */

static bool
examine_argument (machine_mode mode, const_tree type, int in_return,
		  int *int_nregs, int *sse_nregs)
{
  enum x86_64_reg_class regclass[MAX_CLASSES];
  int n = classify_argument (mode, type, regclass, 0);

  *int_nregs = 0;
  *sse_nregs = 0;

  if (!n)
    return true;
  for (n--; n >= 0; n--)
    switch (regclass[n])
      {
      case X86_64_INTEGER_CLASS:
      case X86_64_INTEGERSI_CLASS:
	(*int_nregs)++;
	break;
      case X86_64_SSE_CLASS:
      case X86_64_SSEHF_CLASS:
      case X86_64_SSESF_CLASS:
      case X86_64_SSEDF_CLASS:
	(*sse_nregs)++;
	break;
      case X86_64_NO_CLASS:
      case X86_64_SSEUP_CLASS:
	break;
      case X86_64_X87_CLASS:
      case X86_64_X87UP_CLASS:
      case X86_64_COMPLEX_X87_CLASS:
	if (!in_return)
	  return true;
	break;
      case X86_64_MEMORY_CLASS:
	gcc_unreachable ();
      }

  return false;
}

/* Construct container for the argument used by GCC interface.  See
   FUNCTION_ARG for the detailed description.  */

static rtx
construct_container (machine_mode mode, machine_mode orig_mode,
		     const_tree type, int in_return, int nintregs, int nsseregs,
		     const int *intreg, int sse_regno)
{
  /* The following variables hold the static issued_error state.  */
  static bool issued_sse_arg_error;
  static bool issued_sse_ret_error;
  static bool issued_x87_ret_error;

  machine_mode tmpmode;
  int bytes
    = mode == BLKmode ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
  enum x86_64_reg_class regclass[MAX_CLASSES];
  int n;
  int i;
  int nexps = 0;
  int needed_sseregs, needed_intregs;
  rtx exp[MAX_CLASSES];
  rtx ret;

  n = classify_argument (mode, type, regclass, 0);
  if (!n)
    return NULL;
  if (examine_argument (mode, type, in_return, &needed_intregs,
			&needed_sseregs))
    return NULL;
  if (needed_intregs > nintregs || needed_sseregs > nsseregs)
    return NULL;

  /* We allowed the user to turn off SSE for kernel mode.  Don't crash if
     some less clueful developer tries to use floating-point anyway.  */
  if (needed_sseregs
      && (!TARGET_SSE || (VALID_SSE2_TYPE_MODE (mode) && !TARGET_SSE2)))
    {
      /* Return early if we shouldn't raise an error for invalid
	 calls.  */
      if (cfun != NULL && cfun->machine->silent_p)
	return NULL;
      if (in_return)
	{
	  if (!issued_sse_ret_error)
	    {
	      if (VALID_SSE2_TYPE_MODE (mode))
		error ("SSE register return with SSE2 disabled");
	      else
		error ("SSE register return with SSE disabled");
	      issued_sse_ret_error = true;
	    }
	}
      else if (!issued_sse_arg_error)
	{
	  if (VALID_SSE2_TYPE_MODE (mode))
	    error ("SSE register argument with SSE2 disabled");
	  else
	    error ("SSE register argument with SSE disabled");
	  issued_sse_arg_error = true;
	}
      return NULL;
    }

  /* Likewise, error if the ABI requires us to return values in the
     x87 registers and the user specified -mno-80387.  */
  if (!TARGET_FLOAT_RETURNS_IN_80387 && in_return)
    for (i = 0; i < n; i++)
      if (regclass[i] == X86_64_X87_CLASS
	  || regclass[i] == X86_64_X87UP_CLASS
	  || regclass[i] == X86_64_COMPLEX_X87_CLASS)
	{
	  /* Return early if we shouldn't raise an error for invalid
	     calls.  */
	  if (cfun != NULL && cfun->machine->silent_p)
	    return NULL;
	  if (!issued_x87_ret_error)
	    {
	      error ("x87 register return with x87 disabled");
	      issued_x87_ret_error = true;
	    }
	  return NULL;
	}

  /* First construct simple cases.  Avoid SCmode, since we want to use
     single register to pass this type.  */
  if (n == 1 && mode != SCmode && mode != HCmode)
    switch (regclass[0])
      {
      case X86_64_INTEGER_CLASS:
      case X86_64_INTEGERSI_CLASS:
	return gen_rtx_REG (mode, intreg[0]);
      case X86_64_SSE_CLASS:
      case X86_64_SSEHF_CLASS:
      case X86_64_SSESF_CLASS:
      case X86_64_SSEDF_CLASS:
	if (mode != BLKmode)
	  return gen_reg_or_parallel (mode, orig_mode,
				      GET_SSE_REGNO (sse_regno));
	break;
      case X86_64_X87_CLASS:
      case X86_64_COMPLEX_X87_CLASS:
	return gen_rtx_REG (mode, FIRST_STACK_REG);
      case X86_64_NO_CLASS:
	/* Zero sized array, struct or class.  */
	return NULL;
      default:
	gcc_unreachable ();
      }
  if (n == 2
      && regclass[0] == X86_64_SSE_CLASS
      && regclass[1] == X86_64_SSEUP_CLASS
      && mode != BLKmode)
    return gen_reg_or_parallel (mode, orig_mode,
				GET_SSE_REGNO (sse_regno));
  if (n == 4
      && regclass[0] == X86_64_SSE_CLASS
      && regclass[1] == X86_64_SSEUP_CLASS
      && regclass[2] == X86_64_SSEUP_CLASS
      && regclass[3] == X86_64_SSEUP_CLASS
      && mode != BLKmode)
    return gen_reg_or_parallel (mode, orig_mode,
				GET_SSE_REGNO (sse_regno));
  if (n == 8
      && regclass[0] == X86_64_SSE_CLASS
      && regclass[1] == X86_64_SSEUP_CLASS
      && regclass[2] == X86_64_SSEUP_CLASS
      && regclass[3] == X86_64_SSEUP_CLASS
      && regclass[4] == X86_64_SSEUP_CLASS
      && regclass[5] == X86_64_SSEUP_CLASS
      && regclass[6] == X86_64_SSEUP_CLASS
      && regclass[7] == X86_64_SSEUP_CLASS
      && mode != BLKmode)
    return gen_reg_or_parallel (mode, orig_mode,
				GET_SSE_REGNO (sse_regno));
  if (n == 2
      && regclass[0] == X86_64_X87_CLASS
      && regclass[1] == X86_64_X87UP_CLASS)
    return gen_rtx_REG (XFmode, FIRST_STACK_REG);

  if (n == 2
      && regclass[0] == X86_64_INTEGER_CLASS
      && regclass[1] == X86_64_INTEGER_CLASS
      && (mode == CDImode || mode == TImode || mode == BLKmode)
      && intreg[0] + 1 == intreg[1])
    {
      if (mode == BLKmode)
	{
	  /* Use TImode for BLKmode values in 2 integer registers.  */
	  exp[0] = gen_rtx_EXPR_LIST (VOIDmode,
				      gen_rtx_REG (TImode, intreg[0]),
				      GEN_INT (0));
	  ret = gen_rtx_PARALLEL (mode, rtvec_alloc (1));
	  XVECEXP (ret, 0, 0) = exp[0];
	  return ret;
	}
      else
	return gen_rtx_REG (mode, intreg[0]);
    }

  /* Otherwise figure out the entries of the PARALLEL.  */
  for (i = 0; i < n; i++)
    {
      int pos;

      switch (regclass[i])
        {
	  case X86_64_NO_CLASS:
	    break;
	  case X86_64_INTEGER_CLASS:
	  case X86_64_INTEGERSI_CLASS:
	    /* Merge TImodes on aligned occasions here too.  */
	    if (i * 8 + 8 > bytes)
	      {
		unsigned int tmpbits = (bytes - i * 8) * BITS_PER_UNIT;
		if (!int_mode_for_size (tmpbits, 0).exists (&tmpmode))
		  /* We've requested 24 bytes we
		     don't have mode for.  Use DImode.  */
		  tmpmode = DImode;
	      }
	    else if (regclass[i] == X86_64_INTEGERSI_CLASS)
	      tmpmode = SImode;
	    else
	      tmpmode = DImode;
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (tmpmode, *intreg),
				   GEN_INT (i*8));
	    intreg++;
	    break;
	  case X86_64_SSEHF_CLASS:
	    tmpmode = (mode == BFmode ? BFmode : HFmode);
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (tmpmode,
						GET_SSE_REGNO (sse_regno)),
				   GEN_INT (i*8));
	    sse_regno++;
	    break;
	  case X86_64_SSESF_CLASS:
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (SFmode,
						GET_SSE_REGNO (sse_regno)),
				   GEN_INT (i*8));
	    sse_regno++;
	    break;
	  case X86_64_SSEDF_CLASS:
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (DFmode,
						GET_SSE_REGNO (sse_regno)),
				   GEN_INT (i*8));
	    sse_regno++;
	    break;
	  case X86_64_SSE_CLASS:
	    pos = i;
	    switch (n)
	      {
	      case 1:
		tmpmode = DImode;
		break;
	      case 2:
		if (i == 0 && regclass[1] == X86_64_SSEUP_CLASS)
		  {
		    tmpmode = TImode;
		    i++;
		  }
		else
		  tmpmode = DImode;
		break;
	      case 4:
		gcc_assert (i == 0
			    && regclass[1] == X86_64_SSEUP_CLASS
			    && regclass[2] == X86_64_SSEUP_CLASS
			    && regclass[3] == X86_64_SSEUP_CLASS);
		tmpmode = OImode;
		i += 3;
		break;
	      case 8:
		gcc_assert (i == 0
			    && regclass[1] == X86_64_SSEUP_CLASS
			    && regclass[2] == X86_64_SSEUP_CLASS
			    && regclass[3] == X86_64_SSEUP_CLASS
			    && regclass[4] == X86_64_SSEUP_CLASS
			    && regclass[5] == X86_64_SSEUP_CLASS
			    && regclass[6] == X86_64_SSEUP_CLASS
			    && regclass[7] == X86_64_SSEUP_CLASS);
		tmpmode = XImode;
		i += 7;
		break;
	      default:
		gcc_unreachable ();
	      }
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (tmpmode,
						GET_SSE_REGNO (sse_regno)),
				   GEN_INT (pos*8));
	    sse_regno++;
	    break;
	  default:
	    gcc_unreachable ();
	}
    }

  /* Empty aligned struct, union or class.  */
  if (nexps == 0)
    return NULL;

  ret =  gen_rtx_PARALLEL (mode, rtvec_alloc (nexps));
  for (i = 0; i < nexps; i++)
    XVECEXP (ret, 0, i) = exp [i];
  return ret;
}

/* Update the data in CUM to advance over an argument of mode MODE
   and data type TYPE.  (TYPE is null for libcalls where that information
   may not be available.)

   Return a number of integer regsiters advanced over.  */

static int
function_arg_advance_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
			 const_tree type, HOST_WIDE_INT bytes,
			 HOST_WIDE_INT words)
{
  int res = 0;
  bool error_p = false;

  if (TARGET_IAMCU)
    {
      /* Intel MCU psABI passes scalars and aggregates no larger than 8
	 bytes in registers.  */
      if (!VECTOR_MODE_P (mode) && bytes <= 8)
	goto pass_in_reg;
      return res;
    }

  switch (mode)
    {
    default:
      break;

    case E_BLKmode:
      if (bytes < 0)
	break;
      /* FALLTHRU */

    case E_DImode:
    case E_SImode:
    case E_HImode:
    case E_QImode:
pass_in_reg:
      cum->words += words;
      cum->nregs -= words;
      cum->regno += words;
      if (cum->nregs >= 0)
	res = words;
      if (cum->nregs <= 0)
	{
	  cum->nregs = 0;
	  cfun->machine->arg_reg_available = false;
	  cum->regno = 0;
	}
      break;

    case E_OImode:
      /* OImode shouldn't be used directly.  */
      gcc_unreachable ();

    case E_DFmode:
      if (cum->float_in_sse == -1)
	error_p = true;
      if (cum->float_in_sse < 2)
	break;
      /* FALLTHRU */
    case E_SFmode:
      if (cum->float_in_sse == -1)
	error_p = true;
      if (cum->float_in_sse < 1)
	break;
      /* FALLTHRU */

    case E_V16HFmode:
    case E_V16BFmode:
    case E_V8SFmode:
    case E_V8SImode:
    case E_V64QImode:
    case E_V32HImode:
    case E_V16SImode:
    case E_V8DImode:
    case E_V32HFmode:
    case E_V32BFmode:
    case E_V16SFmode:
    case E_V8DFmode:
    case E_V32QImode:
    case E_V16HImode:
    case E_V4DFmode:
    case E_V4DImode:
    case E_TImode:
    case E_V16QImode:
    case E_V8HImode:
    case E_V4SImode:
    case E_V2DImode:
    case E_V8HFmode:
    case E_V8BFmode:
    case E_V4SFmode:
    case E_V2DFmode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  cum->sse_words += words;
	  cum->sse_nregs -= 1;
	  cum->sse_regno += 1;
	  if (cum->sse_nregs <= 0)
	    {
	      cum->sse_nregs = 0;
	      cum->sse_regno = 0;
	    }
	}
      break;

    case E_V8QImode:
    case E_V4HImode:
    case E_V4HFmode:
    case E_V4BFmode:
    case E_V2SImode:
    case E_V2SFmode:
    case E_V1TImode:
    case E_V1DImode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  cum->mmx_words += words;
	  cum->mmx_nregs -= 1;
	  cum->mmx_regno += 1;
	  if (cum->mmx_nregs <= 0)
	    {
	      cum->mmx_nregs = 0;
	      cum->mmx_regno = 0;
	    }
	}
      break;
    }
  if (error_p)
    {
      cum->float_in_sse = 0;
      error ("calling %qD with SSE calling convention without "
	     "SSE/SSE2 enabled", cum->decl);
      sorry ("this is a GCC bug that can be worked around by adding "
	     "attribute used to function called");
    }

  return res;
}

static int
function_arg_advance_64 (CUMULATIVE_ARGS *cum, machine_mode mode,
			 const_tree type, HOST_WIDE_INT words, bool named)
{
  int int_nregs, sse_nregs;

  /* Unnamed 512 and 256bit vector mode parameters are passed on stack.  */
  if (!named && (VALID_AVX512F_REG_MODE (mode)
		 || VALID_AVX256_REG_MODE (mode)))
    return 0;

  if (!examine_argument (mode, type, 0, &int_nregs, &sse_nregs)
      && sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs)
    {
      cum->nregs -= int_nregs;
      cum->sse_nregs -= sse_nregs;
      cum->regno += int_nregs;
      cum->sse_regno += sse_nregs;
      return int_nregs;
    }
  else
    {
      int align = ix86_function_arg_boundary (mode, type) / BITS_PER_WORD;
      cum->words = ROUND_UP (cum->words, align);
      cum->words += words;
      return 0;
    }
}

static int
function_arg_advance_ms_64 (CUMULATIVE_ARGS *cum, HOST_WIDE_INT bytes,
			    HOST_WIDE_INT words)
{
  /* Otherwise, this should be passed indirect.  */
  gcc_assert (bytes == 1 || bytes == 2 || bytes == 4 || bytes == 8);

  cum->words += words;
  if (cum->nregs > 0)
    {
      cum->nregs -= 1;
      cum->regno += 1;
      return 1;
    }
  return 0;
}

/* Update the data in CUM to advance over argument ARG.  */

static void
ix86_function_arg_advance (cumulative_args_t cum_v,
			   const function_arg_info &arg)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
  machine_mode mode = arg.mode;
  HOST_WIDE_INT bytes, words;
  int nregs;

  /* The argument of interrupt handler is a special case and is
     handled in ix86_function_arg.  */
  if (!cum->caller && cfun->machine->func_type != TYPE_NORMAL)
    return;

  bytes = arg.promoted_size_in_bytes ();
  words = CEIL (bytes, UNITS_PER_WORD);

  if (arg.type)
    mode = type_natural_mode (arg.type, NULL, false);

  if (TARGET_64BIT)
    {
      enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;

      if (call_abi == MS_ABI)
	nregs = function_arg_advance_ms_64 (cum, bytes, words);
      else
	nregs = function_arg_advance_64 (cum, mode, arg.type, words,
					 arg.named);
    }
  else
    nregs = function_arg_advance_32 (cum, mode, arg.type, bytes, words);

  if (!nregs)
    {
      /* Track if there are outgoing arguments on stack.  */
      if (cum->caller)
	cfun->machine->outgoing_args_on_stack = true;
    }
}

/* Define where to put the arguments to a function.
   Value is zero to push the argument on the stack,
   or a hard register in which to store the argument.

   MODE is the argument's machine mode.
   TYPE is the data type of the argument (as a tree).
    This is null for libcalls where that information may
    not be available.
   CUM is a variable of type CUMULATIVE_ARGS which gives info about
    the preceding args and about the function being called.
   NAMED is nonzero if this argument is a named parameter
    (otherwise it is an extra parameter matching an ellipsis).  */

static rtx
function_arg_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
		 machine_mode orig_mode, const_tree type,
		 HOST_WIDE_INT bytes, HOST_WIDE_INT words)
{
  bool error_p = false;

  /* Avoid the AL settings for the Unix64 ABI.  */
  if (mode == VOIDmode)
    return constm1_rtx;

  if (TARGET_IAMCU)
    {
      /* Intel MCU psABI passes scalars and aggregates no larger than 8
	 bytes in registers.  */
      if (!VECTOR_MODE_P (mode) && bytes <= 8)
	goto pass_in_reg;
      return NULL_RTX;
    }

  switch (mode)
    {
    default:
      break;

    case E_BLKmode:
      if (bytes < 0)
	break;
      /* FALLTHRU */
    case E_DImode:
    case E_SImode:
    case E_HImode:
    case E_QImode:
pass_in_reg:
      if (words <= cum->nregs)
	{
	  int regno = cum->regno;

	  /* Fastcall allocates the first two DWORD (SImode) or
            smaller arguments to ECX and EDX if it isn't an
            aggregate type .  */
	  if (cum->fastcall)
	    {
	      if (mode == BLKmode
		  || mode == DImode
		  || (type && AGGREGATE_TYPE_P (type)))
	        break;

	      /* ECX not EAX is the first allocated register.  */
	      if (regno == AX_REG)
		regno = CX_REG;
	    }
	  return gen_rtx_REG (mode, regno);
	}
      break;

    case E_DFmode:
      if (cum->float_in_sse == -1)
	error_p = true;
      if (cum->float_in_sse < 2)
	break;
      /* FALLTHRU */
    case E_SFmode:
      if (cum->float_in_sse == -1)
	error_p = true;
      if (cum->float_in_sse < 1)
	break;
      /* FALLTHRU */
    case E_TImode:
      /* In 32bit, we pass TImode in xmm registers.  */
    case E_V16QImode:
    case E_V8HImode:
    case E_V4SImode:
    case E_V2DImode:
    case E_V8HFmode:
    case E_V8BFmode:
    case E_V4SFmode:
    case E_V2DFmode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  if (cum->sse_nregs)
	    return gen_reg_or_parallel (mode, orig_mode,
				        cum->sse_regno + FIRST_SSE_REG);
	}
      break;

    case E_OImode:
    case E_XImode:
      /* OImode and XImode shouldn't be used directly.  */
      gcc_unreachable ();

    case E_V64QImode:
    case E_V32HImode:
    case E_V16SImode:
    case E_V8DImode:
    case E_V32HFmode:
    case E_V32BFmode:
    case E_V16SFmode:
    case E_V8DFmode:
    case E_V16HFmode:
    case E_V16BFmode:
    case E_V8SFmode:
    case E_V8SImode:
    case E_V32QImode:
    case E_V16HImode:
    case E_V4DFmode:
    case E_V4DImode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  if (cum->sse_nregs)
	    return gen_reg_or_parallel (mode, orig_mode,
				        cum->sse_regno + FIRST_SSE_REG);
	}
      break;

    case E_V8QImode:
    case E_V4HImode:
    case E_V4HFmode:
    case E_V4BFmode:
    case E_V2SImode:
    case E_V2SFmode:
    case E_V1TImode:
    case E_V1DImode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  if (cum->mmx_nregs)
	    return gen_reg_or_parallel (mode, orig_mode,
				        cum->mmx_regno + FIRST_MMX_REG);
	}
      break;
    }
  if (error_p)
    {
      cum->float_in_sse = 0;
      error ("calling %qD with SSE calling convention without "
	     "SSE/SSE2 enabled", cum->decl);
      sorry ("this is a GCC bug that can be worked around by adding "
	     "attribute used to function called");
    }

  return NULL_RTX;
}

static rtx
function_arg_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
		 machine_mode orig_mode, const_tree type, bool named)
{
  /* Handle a hidden AL argument containing number of registers
     for varargs x86-64 functions.  */
  if (mode == VOIDmode)
    return GEN_INT (cum->maybe_vaarg
		    ? (cum->sse_nregs < 0
		       ? X86_64_SSE_REGPARM_MAX
		       : cum->sse_regno)
		    : -1);

  switch (mode)
    {
    default:
      break;

    case E_V16HFmode:
    case E_V16BFmode:
    case E_V8SFmode:
    case E_V8SImode:
    case E_V32QImode:
    case E_V16HImode:
    case E_V4DFmode:
    case E_V4DImode:
    case E_V32HFmode:
    case E_V32BFmode:
    case E_V16SFmode:
    case E_V16SImode:
    case E_V64QImode:
    case E_V32HImode:
    case E_V8DFmode:
    case E_V8DImode:
      /* Unnamed 256 and 512bit vector mode parameters are passed on stack.  */
      if (!named)
	return NULL;
      break;
    }

  return construct_container (mode, orig_mode, type, 0, cum->nregs,
			      cum->sse_nregs,
			      &x86_64_int_parameter_registers [cum->regno],
			      cum->sse_regno);
}

static rtx
function_arg_ms_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
		    machine_mode orig_mode, bool named, const_tree type,
		    HOST_WIDE_INT bytes)
{
  unsigned int regno;

  /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
     We use value of -2 to specify that current function call is MSABI.  */
  if (mode == VOIDmode)
    return GEN_INT (-2);

  /* If we've run out of registers, it goes on the stack.  */
  if (cum->nregs == 0)
    return NULL_RTX;

  regno = x86_64_ms_abi_int_parameter_registers[cum->regno];

  /* Only floating point modes are passed in anything but integer regs.  */
  if (TARGET_SSE && (mode == SFmode || mode == DFmode))
    {
      if (named)
	{
	  if (type == NULL_TREE || !AGGREGATE_TYPE_P (type))
	    regno = cum->regno + FIRST_SSE_REG;
	}
      else
	{
	  rtx t1, t2;

	  /* Unnamed floating parameters are passed in both the
	     SSE and integer registers.  */
	  t1 = gen_rtx_REG (mode, cum->regno + FIRST_SSE_REG);
	  t2 = gen_rtx_REG (mode, regno);
	  t1 = gen_rtx_EXPR_LIST (VOIDmode, t1, const0_rtx);
	  t2 = gen_rtx_EXPR_LIST (VOIDmode, t2, const0_rtx);
	  return gen_rtx_PARALLEL (mode, gen_rtvec (2, t1, t2));
	}
    }
  /* Handle aggregated types passed in register.  */
  if (orig_mode == BLKmode)
    {
      if (bytes > 0 && bytes <= 8)
        mode = (bytes > 4 ? DImode : SImode);
      if (mode == BLKmode)
        mode = DImode;
    }

  return gen_reg_or_parallel (mode, orig_mode, regno);
}

/* Return where to put the arguments to a function.
   Return zero to push the argument on the stack, or a hard register in which to store the argument.

   ARG describes the argument while CUM gives information about the
   preceding args and about the function being called.  */

static rtx
ix86_function_arg (cumulative_args_t cum_v, const function_arg_info &arg)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
  machine_mode mode = arg.mode;
  HOST_WIDE_INT bytes, words;
  rtx reg;

  if (!cum->caller && cfun->machine->func_type != TYPE_NORMAL)
    {
      gcc_assert (arg.type != NULL_TREE);
      if (POINTER_TYPE_P (arg.type))
	{
	  /* This is the pointer argument.  */
	  gcc_assert (TYPE_MODE (arg.type) == ptr_mode);
	  /* It is at -WORD(AP) in the current frame in interrupt and
	     exception handlers.  */
	  reg = plus_constant (Pmode, arg_pointer_rtx, -UNITS_PER_WORD);
	}
      else
	{
	  gcc_assert (cfun->machine->func_type == TYPE_EXCEPTION
		      && TREE_CODE (arg.type) == INTEGER_TYPE
		      && TYPE_MODE (arg.type) == word_mode);
	  /* The error code is the word-mode integer argument at
	     -2 * WORD(AP) in the current frame of the exception
	     handler.  */
	  reg = gen_rtx_MEM (word_mode,
			     plus_constant (Pmode,
					    arg_pointer_rtx,
					    -2 * UNITS_PER_WORD));
	}
      return reg;
    }

  bytes = arg.promoted_size_in_bytes ();
  words = CEIL (bytes, UNITS_PER_WORD);

  /* To simplify the code below, represent vector types with a vector mode
     even if MMX/SSE are not active.  */
  if (arg.type && VECTOR_TYPE_P (arg.type))
    mode = type_natural_mode (arg.type, cum, false);

  if (TARGET_64BIT)
    {
      enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;

      if (call_abi == MS_ABI)
	reg = function_arg_ms_64 (cum, mode, arg.mode, arg.named,
				  arg.type, bytes);
      else
	reg = function_arg_64 (cum, mode, arg.mode, arg.type, arg.named);
    }
  else
    reg = function_arg_32 (cum, mode, arg.mode, arg.type, bytes, words);

  /* Track if there are outgoing arguments on stack.  */
  if (reg == NULL_RTX && cum->caller)
    cfun->machine->outgoing_args_on_stack = true;

  return reg;
}

/* A C expression that indicates when an argument must be passed by
   reference.  If nonzero for an argument, a copy of that argument is
   made in memory and a pointer to the argument is passed instead of
   the argument itself.  The pointer is passed in whatever way is
   appropriate for passing a pointer to that type.  */

static bool
ix86_pass_by_reference (cumulative_args_t cum_v, const function_arg_info &arg)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);

  if (TARGET_64BIT)
    {
      enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;

      /* See Windows x64 Software Convention.  */
      if (call_abi == MS_ABI)
	{
	  HOST_WIDE_INT msize = GET_MODE_SIZE (arg.mode);

	  if (tree type = arg.type)
	    {
	      /* Arrays are passed by reference.  */
	      if (TREE_CODE (type) == ARRAY_TYPE)
		return true;

	      if (RECORD_OR_UNION_TYPE_P (type))
		{
		  /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
		     are passed by reference.  */
		  msize = int_size_in_bytes (type);
		}
	    }

	  /* __m128 is passed by reference.  */
	  return msize != 1 && msize != 2 && msize != 4 && msize != 8;
	}
      else if (arg.type && int_size_in_bytes (arg.type) == -1)
	return true;
    }

  return false;
}

/* Return true when TYPE should be 128bit aligned for 32bit argument
   passing ABI.  XXX: This function is obsolete and is only used for
   checking psABI compatibility with previous versions of GCC.  */

static bool
ix86_compat_aligned_value_p (const_tree type)
{
  machine_mode mode = TYPE_MODE (type);
  if (((TARGET_SSE && SSE_REG_MODE_P (mode))
       || mode == TDmode
       || mode == TFmode
       || mode == TCmode)
      && (!TYPE_USER_ALIGN (type) || TYPE_ALIGN (type) > 128))
    return true;
  if (TYPE_ALIGN (type) < 128)
    return false;

  if (AGGREGATE_TYPE_P (type))
    {
      /* Walk the aggregates recursively.  */
      switch (TREE_CODE (type))
	{
	case RECORD_TYPE:
	case UNION_TYPE:
	case QUAL_UNION_TYPE:
	  {
	    tree field;

	    /* Walk all the structure fields.  */
	    for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
	      {
		if (TREE_CODE (field) == FIELD_DECL
		    && ix86_compat_aligned_value_p (TREE_TYPE (field)))
		  return true;
	      }
	    break;
	  }

	case ARRAY_TYPE:
	  /* Just for use if some languages passes arrays by value.  */
	  if (ix86_compat_aligned_value_p (TREE_TYPE (type)))
	    return true;
	  break;

	default:
	  gcc_unreachable ();
	}
    }
  return false;
}

/* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
   XXX: This function is obsolete and is only used for checking psABI
   compatibility with previous versions of GCC.  */

static unsigned int
ix86_compat_function_arg_boundary (machine_mode mode,
				   const_tree type, unsigned int align)
{
  /* In 32bit, only _Decimal128 and __float128 are aligned to their
     natural boundaries.  */
  if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
    {
      /* i386 ABI defines all arguments to be 4 byte aligned.  We have to
	 make an exception for SSE modes since these require 128bit
	 alignment.

	 The handling here differs from field_alignment.  ICC aligns MMX
	 arguments to 4 byte boundaries, while structure fields are aligned
	 to 8 byte boundaries.  */
      if (!type)
	{
	  if (!(TARGET_SSE && SSE_REG_MODE_P (mode)))
	    align = PARM_BOUNDARY;
	}
      else
	{
	  if (!ix86_compat_aligned_value_p (type))
	    align = PARM_BOUNDARY;
	}
    }
  if (align > BIGGEST_ALIGNMENT)
    align = BIGGEST_ALIGNMENT;
  return align;
}

/* Return true when TYPE should be 128bit aligned for 32bit argument
   passing ABI.  */

static bool
ix86_contains_aligned_value_p (const_tree type)
{
  machine_mode mode = TYPE_MODE (type);

  if (mode == XFmode || mode == XCmode)
    return false;

  if (TYPE_ALIGN (type) < 128)
    return false;

  if (AGGREGATE_TYPE_P (type))
    {
      /* Walk the aggregates recursively.  */
      switch (TREE_CODE (type))
	{
	case RECORD_TYPE:
	case UNION_TYPE:
	case QUAL_UNION_TYPE:
	  {
	    tree field;

	    /* Walk all the structure fields.  */
	    for (field = TYPE_FIELDS (type);
		 field;
		 field = DECL_CHAIN (field))
	      {
		if (TREE_CODE (field) == FIELD_DECL
		    && ix86_contains_aligned_value_p (TREE_TYPE (field)))
		  return true;
	      }
	    break;
	  }

	case ARRAY_TYPE:
	  /* Just for use if some languages passes arrays by value.  */
	  if (ix86_contains_aligned_value_p (TREE_TYPE (type)))
	    return true;
	  break;

	default:
	  gcc_unreachable ();
	}
    }
  else
    return TYPE_ALIGN (type) >= 128;

  return false;
}

/* Gives the alignment boundary, in bits, of an argument with the
   specified mode and type.  */

static unsigned int
ix86_function_arg_boundary (machine_mode mode, const_tree type)
{
  unsigned int align;
  if (type)
    {
      /* Since the main variant type is used for call, we convert it to
	 the main variant type.  */
      type = TYPE_MAIN_VARIANT (type);
      align = TYPE_ALIGN (type);
      if (TYPE_EMPTY_P (type))
	return PARM_BOUNDARY;
    }
  else
    align = GET_MODE_ALIGNMENT (mode);
  if (align < PARM_BOUNDARY)
    align = PARM_BOUNDARY;
  else
    {
      static bool warned;
      unsigned int saved_align = align;

      if (!TARGET_64BIT)
	{
	  /* i386 ABI defines XFmode arguments to be 4 byte aligned.  */
	  if (!type)
	    {
	      if (mode == XFmode || mode == XCmode)
		align = PARM_BOUNDARY;
	    }
	  else if (!ix86_contains_aligned_value_p (type))
	    align = PARM_BOUNDARY;

	  if (align < 128)
	    align = PARM_BOUNDARY;
	}

      if (warn_psabi
	  && !warned
	  && align != ix86_compat_function_arg_boundary (mode, type,
							 saved_align))
	{
	  warned = true;
	  inform (input_location,
		  "the ABI for passing parameters with %d-byte"
		  " alignment has changed in GCC 4.6",
		  align / BITS_PER_UNIT);
	}
    }

  return align;
}

/* Return true if N is a possible register number of function value.  */

static bool
ix86_function_value_regno_p (const unsigned int regno)
{
  switch (regno)
    {
    case AX_REG:
      return true;
    case DX_REG:
      return (!TARGET_64BIT || ix86_cfun_abi () != MS_ABI);
    case DI_REG:
    case SI_REG:
      return TARGET_64BIT && ix86_cfun_abi () != MS_ABI;

      /* Complex values are returned in %st(0)/%st(1) pair.  */
    case ST0_REG:
    case ST1_REG:
      /* TODO: The function should depend on current function ABI but
       builtins.cc would need updating then. Therefore we use the
       default ABI.  */
      if (TARGET_64BIT && ix86_cfun_abi () == MS_ABI)
	return false;
      return TARGET_FLOAT_RETURNS_IN_80387;

      /* Complex values are returned in %xmm0/%xmm1 pair.  */
    case XMM0_REG:
    case XMM1_REG:
      return TARGET_SSE;

    case MM0_REG:
      if (TARGET_MACHO || TARGET_64BIT)
	return false;
      return TARGET_MMX;
    }

  return false;
}

/* Check whether the register REGNO should be zeroed on X86.
   When ALL_SSE_ZEROED is true, all SSE registers have been zeroed
   together, no need to zero it again.
   When NEED_ZERO_MMX is true, MMX registers should be cleared.  */

static bool
zero_call_used_regno_p (const unsigned int regno,
			bool all_sse_zeroed,
			bool need_zero_mmx)
{
  return GENERAL_REGNO_P (regno)
	 || (!all_sse_zeroed && SSE_REGNO_P (regno))
	 || MASK_REGNO_P (regno)
	 || (need_zero_mmx && MMX_REGNO_P (regno));
}

/* Return the machine_mode that is used to zero register REGNO.  */

static machine_mode
zero_call_used_regno_mode (const unsigned int regno)
{
  /* NB: We only need to zero the lower 32 bits for integer registers
     and the lower 128 bits for vector registers since destination are
     zero-extended to the full register width.  */
  if (GENERAL_REGNO_P (regno))
    return SImode;
  else if (SSE_REGNO_P (regno))
    return V4SFmode;
  else if (MASK_REGNO_P (regno))
    return HImode;
  else if (MMX_REGNO_P (regno))
    return V2SImode;
  else
    gcc_unreachable ();
}

/* Generate a rtx to zero all vector registers together if possible,
   otherwise, return NULL.  */

static rtx
zero_all_vector_registers (HARD_REG_SET need_zeroed_hardregs)
{
  if (!TARGET_AVX)
    return NULL;

  for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if ((LEGACY_SSE_REGNO_P (regno)
	 || (TARGET_64BIT
	     && (REX_SSE_REGNO_P (regno)
		 || (TARGET_AVX512F && EXT_REX_SSE_REGNO_P (regno)))))
	&& !TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
      return NULL;

  return gen_avx_vzeroall ();
}

/* Generate insns to zero all st registers together.
   Return true when zeroing instructions are generated.
   Assume the number of st registers that are zeroed is num_of_st,
   we will emit the following sequence to zero them together:
		  fldz;		\
		  fldz;		\
		  ...
		  fldz;		\
		  fstp %%st(0);	\
		  fstp %%st(0);	\
		  ...
		  fstp %%st(0);
   i.e., num_of_st fldz followed by num_of_st fstp to clear the stack
   mark stack slots empty.

   How to compute the num_of_st:
   There is no direct mapping from stack registers to hard register
   numbers.  If one stack register needs to be cleared, we don't know
   where in the stack the value remains.  So, if any stack register
   needs to be cleared, the whole stack should be cleared.  However,
   x87 stack registers that hold the return value should be excluded.
   x87 returns in the top (two for complex values) register, so
   num_of_st should be 7/6 when x87 returns, otherwise it will be 8.
   return the value of num_of_st.  */


static int
zero_all_st_registers (HARD_REG_SET need_zeroed_hardregs)
{

  /* If the FPU is disabled, no need to zero all st registers.  */
  if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
    return 0;

  unsigned int num_of_st = 0;
  for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if ((STACK_REGNO_P (regno) || MMX_REGNO_P (regno))
	&& TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
      {
	num_of_st++;
	break;
      }

  if (num_of_st == 0)
    return 0;

  bool return_with_x87 = false;
  return_with_x87 = (crtl->return_rtx
		     && (STACK_REG_P (crtl->return_rtx)));

  bool complex_return = false;
  complex_return = (crtl->return_rtx
		    && COMPLEX_MODE_P (GET_MODE (crtl->return_rtx)));

  if (return_with_x87)
    if (complex_return)
      num_of_st = 6;
    else
      num_of_st = 7;
  else
    num_of_st = 8;

  rtx st_reg = gen_rtx_REG (XFmode, FIRST_STACK_REG);
  for (unsigned int i = 0; i < num_of_st; i++)
    emit_insn (gen_rtx_SET (st_reg, CONST0_RTX (XFmode)));

  for (unsigned int i = 0; i < num_of_st; i++)
    {
      rtx insn;
      insn = emit_insn (gen_rtx_SET (st_reg, st_reg));
      add_reg_note (insn, REG_DEAD, st_reg);
    }
  return num_of_st;
}


/* When the routine exit in MMX mode, if any ST register needs
   to be zeroed, we should clear all MMX registers except the
   RET_MMX_REGNO that holds the return value.  */
static bool
zero_all_mm_registers (HARD_REG_SET need_zeroed_hardregs,
		       unsigned int ret_mmx_regno)
{
  bool need_zero_all_mm = false;
  for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (STACK_REGNO_P (regno)
	&& TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
      {
	need_zero_all_mm = true;
	break;
      }

  if (!need_zero_all_mm)
    return false;

  machine_mode mode = V2SImode;
  for (unsigned int regno = FIRST_MMX_REG; regno <= LAST_MMX_REG; regno++)
    if (regno != ret_mmx_regno)
      {
	rtx reg = gen_rtx_REG (mode, regno);
	emit_insn (gen_rtx_SET (reg, CONST0_RTX (mode)));
      }
  return true;
}

/* TARGET_ZERO_CALL_USED_REGS.  */
/* Generate a sequence of instructions that zero registers specified by
   NEED_ZEROED_HARDREGS.  Return the ZEROED_HARDREGS that are actually
   zeroed.  */
static HARD_REG_SET
ix86_zero_call_used_regs (HARD_REG_SET need_zeroed_hardregs)
{
  HARD_REG_SET zeroed_hardregs;
  bool all_sse_zeroed = false;
  int all_st_zeroed_num = 0;
  bool all_mm_zeroed = false;

  CLEAR_HARD_REG_SET (zeroed_hardregs);

  /* first, let's see whether we can zero all vector registers together.  */
  rtx zero_all_vec_insn = zero_all_vector_registers (need_zeroed_hardregs);
  if (zero_all_vec_insn)
    {
      emit_insn (zero_all_vec_insn);
      all_sse_zeroed = true;
    }

  /* mm/st registers are shared registers set, we should follow the following
     rules to clear them:
			MMX exit mode	      x87 exit mode
	-------------|----------------------|---------------
	uses x87 reg | clear all MMX	    | clear all x87
	uses MMX reg | clear individual MMX | clear all x87
	x87 + MMX    | clear all MMX	    | clear all x87

     first, we should decide which mode (MMX mode or x87 mode) the function
     exit with.  */

  bool exit_with_mmx_mode = (crtl->return_rtx
			     && (MMX_REG_P (crtl->return_rtx)));

  if (!exit_with_mmx_mode)
    /* x87 exit mode, we should zero all st registers together.  */
    {
      all_st_zeroed_num = zero_all_st_registers (need_zeroed_hardregs);

      if (all_st_zeroed_num > 0)
	for (unsigned int regno = FIRST_STACK_REG; regno <= LAST_STACK_REG; regno++)
	  /* x87 stack registers that hold the return value should be excluded.
	     x87 returns in the top (two for complex values) register.  */
	  if (all_st_zeroed_num == 8
	      || !((all_st_zeroed_num >= 6 && regno == REGNO (crtl->return_rtx))
		   || (all_st_zeroed_num == 6
		       && (regno == (REGNO (crtl->return_rtx) + 1)))))
	    SET_HARD_REG_BIT (zeroed_hardregs, regno);
    }
  else
    /* MMX exit mode, check whether we can zero all mm registers.  */
    {
      unsigned int exit_mmx_regno = REGNO (crtl->return_rtx);
      all_mm_zeroed = zero_all_mm_registers (need_zeroed_hardregs,
					     exit_mmx_regno);
      if (all_mm_zeroed)
	for (unsigned int regno = FIRST_MMX_REG; regno <= LAST_MMX_REG; regno++)
	  if (regno != exit_mmx_regno)
	    SET_HARD_REG_BIT (zeroed_hardregs, regno);
    }

  /* Now, generate instructions to zero all the other registers.  */

  for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    {
      if (!TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
	continue;
      if (!zero_call_used_regno_p (regno, all_sse_zeroed,
				   exit_with_mmx_mode && !all_mm_zeroed))
	continue;

      SET_HARD_REG_BIT (zeroed_hardregs, regno);

      machine_mode mode = zero_call_used_regno_mode (regno);

      rtx reg = gen_rtx_REG (mode, regno);
      rtx tmp = gen_rtx_SET (reg, CONST0_RTX (mode));

      switch (mode)
	{
	case E_SImode:
	  if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
	    {
	      rtx clob = gen_rtx_CLOBBER (VOIDmode,
					  gen_rtx_REG (CCmode,
						       FLAGS_REG));
	      tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2,
							   tmp,
							   clob));
	    }
	  /* FALLTHRU.  */

	case E_V4SFmode:
	case E_HImode:
	case E_V2SImode:
	  emit_insn (tmp);
	  break;

	default:
	  gcc_unreachable ();
	}
    }
  return zeroed_hardregs;
}

/* Define how to find the value returned by a function.
   VALTYPE is the data type of the value (as a tree).
   If the precise function being called is known, FUNC is its FUNCTION_DECL;
   otherwise, FUNC is 0.  */

static rtx
function_value_32 (machine_mode orig_mode, machine_mode mode,
		   const_tree fntype, const_tree fn)
{
  unsigned int regno;

  /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
     we normally prevent this case when mmx is not available.  However
     some ABIs may require the result to be returned like DImode.  */
  if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
    regno = FIRST_MMX_REG;

  /* 16-byte vector modes in %xmm0.  See ix86_return_in_memory for where
     we prevent this case when sse is not available.  However some ABIs
     may require the result to be returned like integer TImode.  */
  else if (mode == TImode
	   || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
    regno = FIRST_SSE_REG;

  /* 32-byte vector modes in %ymm0.   */
  else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 32)
    regno = FIRST_SSE_REG;

  /* 64-byte vector modes in %zmm0.   */
  else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
    regno = FIRST_SSE_REG;

  /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387).  */
  else if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387)
    regno = FIRST_FLOAT_REG;
  else
    /* Most things go in %eax.  */
    regno = AX_REG;

  /* Return __bf16/ _Float16/_Complex _Foat16 by sse register.  */
  if (mode == HFmode || mode == BFmode)
    {
      if (!TARGET_SSE2)
	{
	  error ("SSE register return with SSE2 disabled");
	  regno = AX_REG;
	}
      else
	regno = FIRST_SSE_REG;
    }

  if (mode == HCmode)
    {
      if (!TARGET_SSE2)
	error ("SSE register return with SSE2 disabled");

      rtx ret = gen_rtx_PARALLEL (mode, rtvec_alloc(1));
      XVECEXP (ret, 0, 0)
	= gen_rtx_EXPR_LIST (VOIDmode,
			     gen_rtx_REG (SImode,
					  TARGET_SSE2 ? FIRST_SSE_REG : AX_REG),
			     GEN_INT (0));
      return ret;
    }

  /* Override FP return register with %xmm0 for local functions when
     SSE math is enabled or for functions with sseregparm attribute.  */
  if ((fn || fntype) && (mode == SFmode || mode == DFmode))
    {
      int sse_level = ix86_function_sseregparm (fntype, fn, false);
      if (sse_level == -1)
	{
	  error ("calling %qD with SSE calling convention without "
		 "SSE/SSE2 enabled", fn);
	  sorry ("this is a GCC bug that can be worked around by adding "
		 "attribute used to function called");
	}
      else if ((sse_level >= 1 && mode == SFmode)
	       || (sse_level == 2 && mode == DFmode))
	regno = FIRST_SSE_REG;
    }

  /* OImode shouldn't be used directly.  */
  gcc_assert (mode != OImode);

  return gen_rtx_REG (orig_mode, regno);
}

static rtx
function_value_64 (machine_mode orig_mode, machine_mode mode,
		   const_tree valtype)
{
  rtx ret;

  /* Handle libcalls, which don't provide a type node.  */
  if (valtype == NULL)
    {
      unsigned int regno;

      switch (mode)
	{
	case E_BFmode:
	case E_HFmode:
	case E_HCmode:
	case E_SFmode:
	case E_SCmode:
	case E_DFmode:
	case E_DCmode:
	case E_TFmode:
	case E_SDmode:
	case E_DDmode:
	case E_TDmode:
	  regno = FIRST_SSE_REG;
	  break;
	case E_XFmode:
	case E_XCmode:
	  regno = FIRST_FLOAT_REG;
	  break;
	case E_TCmode:
	  return NULL;
	default:
	  regno = AX_REG;
	}

      return gen_rtx_REG (mode, regno);
    }
  else if (POINTER_TYPE_P (valtype))
    {
      /* Pointers are always returned in word_mode.  */
      mode = word_mode;
    }

  ret = construct_container (mode, orig_mode, valtype, 1,
			     X86_64_REGPARM_MAX, X86_64_SSE_REGPARM_MAX,
			     x86_64_int_return_registers, 0);

  /* For zero sized structures, construct_container returns NULL, but we
     need to keep rest of compiler happy by returning meaningful value.  */
  if (!ret)
    ret = gen_rtx_REG (orig_mode, AX_REG);

  return ret;
}

static rtx
function_value_ms_32 (machine_mode orig_mode, machine_mode mode,
		      const_tree fntype, const_tree fn, const_tree valtype)
{
  unsigned int regno;

  /* Floating point return values in %st(0)
     (unless -mno-fp-ret-in-387 or aggregate type of up to 8 bytes).  */
  if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387
	   && (GET_MODE_SIZE (mode) > 8
	       || valtype == NULL_TREE || !AGGREGATE_TYPE_P (valtype)))
  {
    regno = FIRST_FLOAT_REG;
    return gen_rtx_REG (orig_mode, regno);
  }
  else
    return function_value_32(orig_mode, mode, fntype,fn);
}

static rtx
function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
		      const_tree valtype)
{
  unsigned int regno = AX_REG;

  if (TARGET_SSE)
    {
      switch (GET_MODE_SIZE (mode))
	{
	case 16:
	  if (valtype != NULL_TREE
	      && !VECTOR_INTEGER_TYPE_P (valtype)
	      && !VECTOR_INTEGER_TYPE_P (valtype)
	      && !INTEGRAL_TYPE_P (valtype)
	      && !VECTOR_FLOAT_TYPE_P (valtype))
	    break;
	  if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
	      && !COMPLEX_MODE_P (mode))
	    regno = FIRST_SSE_REG;
	  break;
	case 8:
	case 4:
	  if (valtype != NULL_TREE && AGGREGATE_TYPE_P (valtype))
	    break;
	  if (mode == SFmode || mode == DFmode)
	    regno = FIRST_SSE_REG;
	  break;
	default:
	  break;
        }
    }
  return gen_rtx_REG (orig_mode, regno);
}

static rtx
ix86_function_value_1 (const_tree valtype, const_tree fntype_or_decl,
		       machine_mode orig_mode, machine_mode mode)
{
  const_tree fn, fntype;

  fn = NULL_TREE;
  if (fntype_or_decl && DECL_P (fntype_or_decl))
    fn = fntype_or_decl;
  fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;
  
  if (ix86_function_type_abi (fntype) == MS_ABI)
    {
      if (TARGET_64BIT)
	return function_value_ms_64 (orig_mode, mode, valtype);
      else
	return function_value_ms_32 (orig_mode, mode, fntype, fn, valtype);
    }
  else if (TARGET_64BIT)
    return function_value_64 (orig_mode, mode, valtype);
  else
    return function_value_32 (orig_mode, mode, fntype, fn);
}

static rtx
ix86_function_value (const_tree valtype, const_tree fntype_or_decl, bool)
{
  machine_mode mode, orig_mode;

  orig_mode = TYPE_MODE (valtype);
  mode = type_natural_mode (valtype, NULL, true);
  return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode);
}

/* Pointer function arguments and return values are promoted to
   word_mode for normal functions.  */

static machine_mode
ix86_promote_function_mode (const_tree type, machine_mode mode,
			    int *punsignedp, const_tree fntype,
			    int for_return)
{
  if (cfun->machine->func_type == TYPE_NORMAL
      && type != NULL_TREE
      && POINTER_TYPE_P (type))
    {
      *punsignedp = POINTERS_EXTEND_UNSIGNED;
      return word_mode;
    }
  return default_promote_function_mode (type, mode, punsignedp, fntype,
					for_return);
}

/* Return true if a structure, union or array with MODE containing FIELD
   should be accessed using BLKmode.  */

static bool
ix86_member_type_forces_blk (const_tree field, machine_mode mode)
{
  /* Union with XFmode must be in BLKmode.  */
  return (mode == XFmode
	  && (TREE_CODE (DECL_FIELD_CONTEXT (field)) == UNION_TYPE
	      || TREE_CODE (DECL_FIELD_CONTEXT (field)) == QUAL_UNION_TYPE));
}

rtx
ix86_libcall_value (machine_mode mode)
{
  return ix86_function_value_1 (NULL, NULL, mode, mode);
}

/* Return true iff type is returned in memory.  */

static bool
ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
{
  const machine_mode mode = type_natural_mode (type, NULL, true);
  HOST_WIDE_INT size;

  if (TARGET_64BIT)
    {
      if (ix86_function_type_abi (fntype) == MS_ABI)
	{
	  size = int_size_in_bytes (type);

	  /* __m128 is returned in xmm0.  */
	  if ((!type || VECTOR_INTEGER_TYPE_P (type)
	       || INTEGRAL_TYPE_P (type)
	       || VECTOR_FLOAT_TYPE_P (type))
	      && (SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
	      && !COMPLEX_MODE_P (mode)
	      && (GET_MODE_SIZE (mode) == 16 || size == 16))
	    return false;

	  /* Otherwise, the size must be exactly in [1248]. */
	  return size != 1 && size != 2 && size != 4 && size != 8;
	}
      else
	{
	  int needed_intregs, needed_sseregs;

	  return examine_argument (mode, type, 1,
				   &needed_intregs, &needed_sseregs);
	}
    }
  else
    {
      size = int_size_in_bytes (type);

      /* Intel MCU psABI returns scalars and aggregates no larger than 8
	 bytes in registers.  */
      if (TARGET_IAMCU)
	return VECTOR_MODE_P (mode) || size < 0 || size > 8;

      if (mode == BLKmode)
	return true;

      if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8)
	return false;

      if (VECTOR_MODE_P (mode) || mode == TImode)
	{
	  /* User-created vectors small enough to fit in EAX.  */
	  if (size < 8)
	    return false;

	  /* Unless ABI prescibes otherwise,
	     MMX/3dNow values are returned in MM0 if available.  */
	     
	  if (size == 8)
	    return TARGET_VECT8_RETURNS || !TARGET_MMX;

	  /* SSE values are returned in XMM0 if available.  */
	  if (size == 16)
	    return !TARGET_SSE;

	  /* AVX values are returned in YMM0 if available.  */
	  if (size == 32)
	    return !TARGET_AVX;

	  /* AVX512F values are returned in ZMM0 if available.  */
	  if (size == 64)
	    return !TARGET_AVX512F;
	}

      if (mode == XFmode)
	return false;

      if (size > 12)
	return true;

      /* OImode shouldn't be used directly.  */
      gcc_assert (mode != OImode);

      return false;
    }
}

/* Implement TARGET_PUSH_ARGUMENT.  */

static bool
ix86_push_argument (unsigned int npush)
{
  /* If SSE2 is available, use vector move to put large argument onto
     stack.  NB:  In 32-bit mode, use 8-byte vector move.  */
  return ((!TARGET_SSE2 || npush < (TARGET_64BIT ? 16 : 8))
	  && TARGET_PUSH_ARGS
	  && !ACCUMULATE_OUTGOING_ARGS);
}


/* Create the va_list data type.  */

static tree
ix86_build_builtin_va_list_64 (void)
{
  tree f_gpr, f_fpr, f_ovf, f_sav, record, type_decl;

  record = lang_hooks.types.make_type (RECORD_TYPE);
  type_decl = build_decl (BUILTINS_LOCATION,
			  TYPE_DECL, get_identifier ("__va_list_tag"), record);

  f_gpr = build_decl (BUILTINS_LOCATION,
		      FIELD_DECL, get_identifier ("gp_offset"),
		      unsigned_type_node);
  f_fpr = build_decl (BUILTINS_LOCATION,
		      FIELD_DECL, get_identifier ("fp_offset"),
		      unsigned_type_node);
  f_ovf = build_decl (BUILTINS_LOCATION,
		      FIELD_DECL, get_identifier ("overflow_arg_area"),
		      ptr_type_node);
  f_sav = build_decl (BUILTINS_LOCATION,
		      FIELD_DECL, get_identifier ("reg_save_area"),
		      ptr_type_node);

  va_list_gpr_counter_field = f_gpr;
  va_list_fpr_counter_field = f_fpr;

  DECL_FIELD_CONTEXT (f_gpr) = record;
  DECL_FIELD_CONTEXT (f_fpr) = record;
  DECL_FIELD_CONTEXT (f_ovf) = record;
  DECL_FIELD_CONTEXT (f_sav) = record;

  TYPE_STUB_DECL (record) = type_decl;
  TYPE_NAME (record) = type_decl;
  TYPE_FIELDS (record) = f_gpr;
  DECL_CHAIN (f_gpr) = f_fpr;
  DECL_CHAIN (f_fpr) = f_ovf;
  DECL_CHAIN (f_ovf) = f_sav;

  layout_type (record);

  TYPE_ATTRIBUTES (record) = tree_cons (get_identifier ("sysv_abi va_list"),
					NULL_TREE, TYPE_ATTRIBUTES (record));

  /* The correct type is an array type of one element.  */
  return build_array_type (record, build_index_type (size_zero_node));
}

/* Setup the builtin va_list data type and for 64-bit the additional
   calling convention specific va_list data types.  */

static tree
ix86_build_builtin_va_list (void)
{
  if (TARGET_64BIT)
    {
      /* Initialize ABI specific va_list builtin types.

	 In lto1, we can encounter two va_list types:
	 - one as a result of the type-merge across TUs, and
	 - the one constructed here.
	 These two types will not have the same TYPE_MAIN_VARIANT, and therefore
	 a type identity check in canonical_va_list_type based on
	 TYPE_MAIN_VARIANT (which we used to have) will not work.
	 Instead, we tag each va_list_type_node with its unique attribute, and
	 look for the attribute in the type identity check in
	 canonical_va_list_type.

	 Tagging sysv_va_list_type_node directly with the attribute is
	 problematic since it's a array of one record, which will degrade into a
	 pointer to record when used as parameter (see build_va_arg comments for
	 an example), dropping the attribute in the process.  So we tag the
	 record instead.  */

      /* For SYSV_ABI we use an array of one record.  */
      sysv_va_list_type_node = ix86_build_builtin_va_list_64 ();
	
      /* For MS_ABI we use plain pointer to argument area.  */
      tree char_ptr_type = build_pointer_type (char_type_node);
      tree attr = tree_cons (get_identifier ("ms_abi va_list"), NULL_TREE,
			     TYPE_ATTRIBUTES (char_ptr_type));
      ms_va_list_type_node = build_type_attribute_variant (char_ptr_type, attr);

      return ((ix86_abi == MS_ABI)
	      ? ms_va_list_type_node
	      : sysv_va_list_type_node);
    }
  else
    {
      /* For i386 we use plain pointer to argument area.  */
      return build_pointer_type (char_type_node);
    }
}

/* Worker function for TARGET_SETUP_INCOMING_VARARGS.  */

static void
setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum)
{
  rtx save_area, mem;
  alias_set_type set;
  int i, max;

  /* GPR size of varargs save area.  */
  if (cfun->va_list_gpr_size)
    ix86_varargs_gpr_size = X86_64_REGPARM_MAX * UNITS_PER_WORD;
  else
    ix86_varargs_gpr_size = 0;

  /* FPR size of varargs save area.  We don't need it if we don't pass
     anything in SSE registers.  */
  if (TARGET_SSE && cfun->va_list_fpr_size)
    ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16;
  else
    ix86_varargs_fpr_size = 0;

  if (! ix86_varargs_gpr_size && ! ix86_varargs_fpr_size)
    return;

  save_area = frame_pointer_rtx;
  set = get_varargs_alias_set ();

  max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
  if (max > X86_64_REGPARM_MAX)
    max = X86_64_REGPARM_MAX;

  for (i = cum->regno; i < max; i++)
    {
      mem = gen_rtx_MEM (word_mode,
			 plus_constant (Pmode, save_area, i * UNITS_PER_WORD));
      MEM_NOTRAP_P (mem) = 1;
      set_mem_alias_set (mem, set);
      emit_move_insn (mem,
		      gen_rtx_REG (word_mode,
				   x86_64_int_parameter_registers[i]));
    }

  if (ix86_varargs_fpr_size)
    {
      machine_mode smode;
      rtx_code_label *label;
      rtx test;

      /* Now emit code to save SSE registers.  The AX parameter contains number
	 of SSE parameter registers used to call this function, though all we
	 actually check here is the zero/non-zero status.  */

      label = gen_label_rtx ();
      test = gen_rtx_EQ (VOIDmode, gen_rtx_REG (QImode, AX_REG), const0_rtx);
      emit_jump_insn (gen_cbranchqi4 (test, XEXP (test, 0), XEXP (test, 1),
				      label));

      /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
	 we used movdqa (i.e. TImode) instead?  Perhaps even better would
	 be if we could determine the real mode of the data, via a hook
	 into pass_stdarg.  Ignore all that for now.  */
      smode = V4SFmode;
      if (crtl->stack_alignment_needed < GET_MODE_ALIGNMENT (smode))
	crtl->stack_alignment_needed = GET_MODE_ALIGNMENT (smode);

      max = cum->sse_regno + cfun->va_list_fpr_size / 16;
      if (max > X86_64_SSE_REGPARM_MAX)
	max = X86_64_SSE_REGPARM_MAX;

      for (i = cum->sse_regno; i < max; ++i)
	{
	  mem = plus_constant (Pmode, save_area,
			       i * 16 + ix86_varargs_gpr_size);
	  mem = gen_rtx_MEM (smode, mem);
	  MEM_NOTRAP_P (mem) = 1;
	  set_mem_alias_set (mem, set);
	  set_mem_align (mem, GET_MODE_ALIGNMENT (smode));

	  emit_move_insn (mem, gen_rtx_REG (smode, GET_SSE_REGNO (i)));
	}

      emit_label (label);
    }
}

static void
setup_incoming_varargs_ms_64 (CUMULATIVE_ARGS *cum)
{
  alias_set_type set = get_varargs_alias_set ();
  int i;

  /* Reset to zero, as there might be a sysv vaarg used
     before.  */
  ix86_varargs_gpr_size = 0;
  ix86_varargs_fpr_size = 0;

  for (i = cum->regno; i < X86_64_MS_REGPARM_MAX; i++)
    {
      rtx reg, mem;

      mem = gen_rtx_MEM (Pmode,
			 plus_constant (Pmode, virtual_incoming_args_rtx,
					i * UNITS_PER_WORD));
      MEM_NOTRAP_P (mem) = 1;
      set_mem_alias_set (mem, set);

      reg = gen_rtx_REG (Pmode, x86_64_ms_abi_int_parameter_registers[i]);
      emit_move_insn (mem, reg);
    }
}

static void
ix86_setup_incoming_varargs (cumulative_args_t cum_v,
			     const function_arg_info &arg,
			     int *, int no_rtl)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
  CUMULATIVE_ARGS next_cum;
  tree fntype;

  /* This argument doesn't appear to be used anymore.  Which is good,
     because the old code here didn't suppress rtl generation.  */
  gcc_assert (!no_rtl);

  if (!TARGET_64BIT)
    return;

  fntype = TREE_TYPE (current_function_decl);

  /* For varargs, we do not want to skip the dummy va_dcl argument.
     For stdargs, we do want to skip the last named argument.  */
  next_cum = *cum;
  if (!TYPE_NO_NAMED_ARGS_STDARG_P (TREE_TYPE (current_function_decl))
      && stdarg_p (fntype))
    ix86_function_arg_advance (pack_cumulative_args (&next_cum), arg);

  if (cum->call_abi == MS_ABI)
    setup_incoming_varargs_ms_64 (&next_cum);
  else
    setup_incoming_varargs_64 (&next_cum);
}

/* Checks if TYPE is of kind va_list char *.  */

static bool
is_va_list_char_pointer (tree type)
{
  tree canonic;

  /* For 32-bit it is always true.  */
  if (!TARGET_64BIT)
    return true;
  canonic = ix86_canonical_va_list_type (type);
  return (canonic == ms_va_list_type_node
          || (ix86_abi == MS_ABI && canonic == va_list_type_node));
}

/* Implement va_start.  */

static void
ix86_va_start (tree valist, rtx nextarg)
{
  HOST_WIDE_INT words, n_gpr, n_fpr;
  tree f_gpr, f_fpr, f_ovf, f_sav;
  tree gpr, fpr, ovf, sav, t;
  tree type;
  rtx ovf_rtx;

  if (flag_split_stack
      && cfun->machine->split_stack_varargs_pointer == NULL_RTX)
    {
      unsigned int scratch_regno;

      /* When we are splitting the stack, we can't refer to the stack
	 arguments using internal_arg_pointer, because they may be on
	 the old stack.  The split stack prologue will arrange to
	 leave a pointer to the old stack arguments in a scratch
	 register, which we here copy to a pseudo-register.  The split
	 stack prologue can't set the pseudo-register directly because
	 it (the prologue) runs before any registers have been saved.  */

      scratch_regno = split_stack_prologue_scratch_regno ();
      if (scratch_regno != INVALID_REGNUM)
	{
	  rtx reg;
	  rtx_insn *seq;

	  reg = gen_reg_rtx (Pmode);
	  cfun->machine->split_stack_varargs_pointer = reg;

	  start_sequence ();
	  emit_move_insn (reg, gen_rtx_REG (Pmode, scratch_regno));
	  seq = get_insns ();
	  end_sequence ();

	  push_topmost_sequence ();
	  emit_insn_after (seq, entry_of_function ());
	  pop_topmost_sequence ();
	}
    }

  /* Only 64bit target needs something special.  */
  if (is_va_list_char_pointer (TREE_TYPE (valist)))
    {
      if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
	std_expand_builtin_va_start (valist, nextarg);
      else
	{
	  rtx va_r, next;

	  va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE);
	  next = expand_binop (ptr_mode, add_optab,
			       cfun->machine->split_stack_varargs_pointer,
			       crtl->args.arg_offset_rtx,
			       NULL_RTX, 0, OPTAB_LIB_WIDEN);
	  convert_move (va_r, next, 0);
	}
      return;
    }

  f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
  f_fpr = DECL_CHAIN (f_gpr);
  f_ovf = DECL_CHAIN (f_fpr);
  f_sav = DECL_CHAIN (f_ovf);

  valist = build_simple_mem_ref (valist);
  TREE_TYPE (valist) = TREE_TYPE (sysv_va_list_type_node);
  /* The following should be folded into the MEM_REF offset.  */
  gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), unshare_expr (valist),
		f_gpr, NULL_TREE);
  fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), unshare_expr (valist),
		f_fpr, NULL_TREE);
  ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), unshare_expr (valist),
		f_ovf, NULL_TREE);
  sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), unshare_expr (valist),
		f_sav, NULL_TREE);

  /* Count number of gp and fp argument registers used.  */
  words = crtl->args.info.words;
  n_gpr = crtl->args.info.regno;
  n_fpr = crtl->args.info.sse_regno;

  if (cfun->va_list_gpr_size)
    {
      type = TREE_TYPE (gpr);
      t = build2 (MODIFY_EXPR, type,
		  gpr, build_int_cst (type, n_gpr * 8));
      TREE_SIDE_EFFECTS (t) = 1;
      expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
    }

  if (TARGET_SSE && cfun->va_list_fpr_size)
    {
      type = TREE_TYPE (fpr);
      t = build2 (MODIFY_EXPR, type, fpr,
		  build_int_cst (type, n_fpr * 16 + 8*X86_64_REGPARM_MAX));
      TREE_SIDE_EFFECTS (t) = 1;
      expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
    }

  /* Find the overflow area.  */
  type = TREE_TYPE (ovf);
  if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
    ovf_rtx = crtl->args.internal_arg_pointer;
  else
    ovf_rtx = cfun->machine->split_stack_varargs_pointer;
  t = make_tree (type, ovf_rtx);
  if (words != 0)
    t = fold_build_pointer_plus_hwi (t, words * UNITS_PER_WORD);

  t = build2 (MODIFY_EXPR, type, ovf, t);
  TREE_SIDE_EFFECTS (t) = 1;
  expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);

  if (ix86_varargs_gpr_size || ix86_varargs_fpr_size)
    {
      /* Find the register save area.
	 Prologue of the function save it right above stack frame.  */
      type = TREE_TYPE (sav);
      t = make_tree (type, frame_pointer_rtx);
      if (!ix86_varargs_gpr_size)
	t = fold_build_pointer_plus_hwi (t, -8 * X86_64_REGPARM_MAX);

      t = build2 (MODIFY_EXPR, type, sav, t);
      TREE_SIDE_EFFECTS (t) = 1;
      expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
    }
}

/* Implement va_arg.  */

static tree
ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
		      gimple_seq *post_p)
{
  static const int intreg[6] = { 0, 1, 2, 3, 4, 5 };
  tree f_gpr, f_fpr, f_ovf, f_sav;
  tree gpr, fpr, ovf, sav, t;
  int size, rsize;
  tree lab_false, lab_over = NULL_TREE;
  tree addr, t2;
  rtx container;
  int indirect_p = 0;
  tree ptrtype;
  machine_mode nat_mode;
  unsigned int arg_boundary;
  unsigned int type_align;

  /* Only 64bit target needs something special.  */
  if (is_va_list_char_pointer (TREE_TYPE (valist)))
    return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);

  f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
  f_fpr = DECL_CHAIN (f_gpr);
  f_ovf = DECL_CHAIN (f_fpr);
  f_sav = DECL_CHAIN (f_ovf);

  gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr),
		valist, f_gpr, NULL_TREE);

  fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
  ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
  sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);

  indirect_p = pass_va_arg_by_reference (type);
  if (indirect_p)
    type = build_pointer_type (type);
  size = arg_int_size_in_bytes (type);
  rsize = CEIL (size, UNITS_PER_WORD);

  nat_mode = type_natural_mode (type, NULL, false);
  switch (nat_mode)
    {
    case E_V16HFmode:
    case E_V16BFmode:
    case E_V8SFmode:
    case E_V8SImode:
    case E_V32QImode:
    case E_V16HImode:
    case E_V4DFmode:
    case E_V4DImode:
    case E_V32HFmode:
    case E_V32BFmode:
    case E_V16SFmode:
    case E_V16SImode:
    case E_V64QImode:
    case E_V32HImode:
    case E_V8DFmode:
    case E_V8DImode:
      /* Unnamed 256 and 512bit vector mode parameters are passed on stack.  */
      if (!TARGET_64BIT_MS_ABI)
	{
	  container = NULL;
	  break;
	}
      /* FALLTHRU */

    default:
      container = construct_container (nat_mode, TYPE_MODE (type),
				       type, 0, X86_64_REGPARM_MAX,
				       X86_64_SSE_REGPARM_MAX, intreg,
				       0);
      break;
    }

  /* Pull the value out of the saved registers.  */

  addr = create_tmp_var (ptr_type_node, "addr");
  type_align = TYPE_ALIGN (type);

  if (container)
    {
      int needed_intregs, needed_sseregs;
      bool need_temp;
      tree int_addr, sse_addr;

      lab_false = create_artificial_label (UNKNOWN_LOCATION);
      lab_over = create_artificial_label (UNKNOWN_LOCATION);

      examine_argument (nat_mode, type, 0, &needed_intregs, &needed_sseregs);

      need_temp = (!REG_P (container)
		   && ((needed_intregs && TYPE_ALIGN (type) > 64)
		       || TYPE_ALIGN (type) > 128));

      /* In case we are passing structure, verify that it is consecutive block
         on the register save area.  If not we need to do moves.  */
      if (!need_temp && !REG_P (container))
	{
	  /* Verify that all registers are strictly consecutive  */
	  if (SSE_REGNO_P (REGNO (XEXP (XVECEXP (container, 0, 0), 0))))
	    {
	      int i;

	      for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
		{
		  rtx slot = XVECEXP (container, 0, i);
		  if (REGNO (XEXP (slot, 0)) != FIRST_SSE_REG + (unsigned int) i
		      || INTVAL (XEXP (slot, 1)) != i * 16)
		    need_temp = true;
		}
	    }
	  else
	    {
	      int i;

	      for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
		{
		  rtx slot = XVECEXP (container, 0, i);
		  if (REGNO (XEXP (slot, 0)) != (unsigned int) i
		      || INTVAL (XEXP (slot, 1)) != i * 8)
		    need_temp = true;
		}
	    }
	}
      if (!need_temp)
	{
	  int_addr = addr;
	  sse_addr = addr;
	}
      else
	{
	  int_addr = create_tmp_var (ptr_type_node, "int_addr");
	  sse_addr = create_tmp_var (ptr_type_node, "sse_addr");
	}

      /* First ensure that we fit completely in registers.  */
      if (needed_intregs)
	{
	  t = build_int_cst (TREE_TYPE (gpr),
			     (X86_64_REGPARM_MAX - needed_intregs + 1) * 8);
	  t = build2 (GE_EXPR, boolean_type_node, gpr, t);
	  t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
	  t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
	  gimplify_and_add (t, pre_p);
	}
      if (needed_sseregs)
	{
	  t = build_int_cst (TREE_TYPE (fpr),
			     (X86_64_SSE_REGPARM_MAX - needed_sseregs + 1) * 16
			     + X86_64_REGPARM_MAX * 8);
	  t = build2 (GE_EXPR, boolean_type_node, fpr, t);
	  t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
	  t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
	  gimplify_and_add (t, pre_p);
	}

      /* Compute index to start of area used for integer regs.  */
      if (needed_intregs)
	{
	  /* int_addr = gpr + sav; */
	  t = fold_build_pointer_plus (sav, gpr);
	  gimplify_assign (int_addr, t, pre_p);
	}
      if (needed_sseregs)
	{
	  /* sse_addr = fpr + sav; */
	  t = fold_build_pointer_plus (sav, fpr);
	  gimplify_assign (sse_addr, t, pre_p);
	}
      if (need_temp)
	{
	  int i, prev_size = 0;
	  tree temp = create_tmp_var (type, "va_arg_tmp");
	  TREE_ADDRESSABLE (temp) = 1;

	  /* addr = &temp; */
	  t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
	  gimplify_assign (addr, t, pre_p);

	  for (i = 0; i < XVECLEN (container, 0); i++)
	    {
	      rtx slot = XVECEXP (container, 0, i);
	      rtx reg = XEXP (slot, 0);
	      machine_mode mode = GET_MODE (reg);
	      tree piece_type;
	      tree addr_type;
	      tree daddr_type;
	      tree src_addr, src;
	      int src_offset;
	      tree dest_addr, dest;
	      int cur_size = GET_MODE_SIZE (mode);

	      gcc_assert (prev_size <= INTVAL (XEXP (slot, 1)));
	      prev_size = INTVAL (XEXP (slot, 1));
	      if (prev_size + cur_size > size)
		{
		  cur_size = size - prev_size;
		  unsigned int nbits = cur_size * BITS_PER_UNIT;
		  if (!int_mode_for_size (nbits, 1).exists (&mode))
		    mode = QImode;
		}
	      piece_type = lang_hooks.types.type_for_mode (mode, 1);
	      if (mode == GET_MODE (reg))
		addr_type = build_pointer_type (piece_type);
	      else
		addr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
							 true);
	      daddr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
							true);

	      if (SSE_REGNO_P (REGNO (reg)))
		{
		  src_addr = sse_addr;
		  src_offset = (REGNO (reg) - FIRST_SSE_REG) * 16;
		}
	      else
		{
		  src_addr = int_addr;
		  src_offset = REGNO (reg) * 8;
		}
	      src_addr = fold_convert (addr_type, src_addr);
	      src_addr = fold_build_pointer_plus_hwi (src_addr, src_offset);

	      dest_addr = fold_convert (daddr_type, addr);
	      dest_addr = fold_build_pointer_plus_hwi (dest_addr, prev_size);
	      if (cur_size == GET_MODE_SIZE (mode))
		{
		  src = build_va_arg_indirect_ref (src_addr);
		  dest = build_va_arg_indirect_ref (dest_addr);

		  gimplify_assign (dest, src, pre_p);
		}
	      else
		{
		  tree copy
		    = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMCPY),
				       3, dest_addr, src_addr,
				       size_int (cur_size));
		  gimplify_and_add (copy, pre_p);
		}
	      prev_size += cur_size;
	    }
	}

      if (needed_intregs)
	{
	  t = build2 (PLUS_EXPR, TREE_TYPE (gpr), gpr,
		      build_int_cst (TREE_TYPE (gpr), needed_intregs * 8));
	  gimplify_assign (gpr, t, pre_p);
	  /* The GPR save area guarantees only 8-byte alignment.  */
	  if (!need_temp)
	    type_align = MIN (type_align, 64);
	}

      if (needed_sseregs)
	{
	  t = build2 (PLUS_EXPR, TREE_TYPE (fpr), fpr,
		      build_int_cst (TREE_TYPE (fpr), needed_sseregs * 16));
	  gimplify_assign (unshare_expr (fpr), t, pre_p);
	}

      gimple_seq_add_stmt (pre_p, gimple_build_goto (lab_over));

      gimple_seq_add_stmt (pre_p, gimple_build_label (lab_false));
    }

  /* ... otherwise out of the overflow area.  */

  /* When we align parameter on stack for caller, if the parameter
     alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
     aligned at MAX_SUPPORTED_STACK_ALIGNMENT.  We will match callee
     here with caller.  */
  arg_boundary = ix86_function_arg_boundary (VOIDmode, type);
  if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
    arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT;

  /* Care for on-stack alignment if needed.  */
  if (arg_boundary <= 64 || size == 0)
    t = ovf;
 else
    {
      HOST_WIDE_INT align = arg_boundary / 8;
      t = fold_build_pointer_plus_hwi (ovf, align - 1);
      t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
		  build_int_cst (TREE_TYPE (t), -align));
    }

  gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue);
  gimplify_assign (addr, t, pre_p);

  t = fold_build_pointer_plus_hwi (t, rsize * UNITS_PER_WORD);
  gimplify_assign (unshare_expr (ovf), t, pre_p);

  if (container)
    gimple_seq_add_stmt (pre_p, gimple_build_label (lab_over));

  type = build_aligned_type (type, type_align);
  ptrtype = build_pointer_type_for_mode (type, ptr_mode, true);
  addr = fold_convert (ptrtype, addr);

  if (indirect_p)
    addr = build_va_arg_indirect_ref (addr);
  return build_va_arg_indirect_ref (addr);
}

/* Return true if OPNUM's MEM should be matched
   in movabs* patterns.  */

bool
ix86_check_movabs (rtx insn, int opnum)
{
  rtx set, mem;

  set = PATTERN (insn);
  if (GET_CODE (set) == PARALLEL)
    set = XVECEXP (set, 0, 0);
  gcc_assert (GET_CODE (set) == SET);
  mem = XEXP (set, opnum);
  while (SUBREG_P (mem))
    mem = SUBREG_REG (mem);
  gcc_assert (MEM_P (mem));
  return volatile_ok || !MEM_VOLATILE_P (mem);
}

/* Return false if INSN contains a MEM with a non-default address space.  */
bool
ix86_check_no_addr_space (rtx insn)
{
  subrtx_var_iterator::array_type array;
  FOR_EACH_SUBRTX_VAR (iter, array, PATTERN (insn), ALL)
    {
      rtx x = *iter;
      if (MEM_P (x) && !ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x)))
	return false;
    }
  return true;
}

/* Initialize the table of extra 80387 mathematical constants.  */

static void
init_ext_80387_constants (void)
{
  static const char * cst[5] =
  {
    "0.3010299956639811952256464283594894482",  /* 0: fldlg2  */
    "0.6931471805599453094286904741849753009",  /* 1: fldln2  */
    "1.4426950408889634073876517827983434472",  /* 2: fldl2e  */
    "3.3219280948873623478083405569094566090",  /* 3: fldl2t  */
    "3.1415926535897932385128089594061862044",  /* 4: fldpi   */
  };
  int i;

  for (i = 0; i < 5; i++)
    {
      real_from_string (&ext_80387_constants_table[i], cst[i]);
      /* Ensure each constant is rounded to XFmode precision.  */
      real_convert (&ext_80387_constants_table[i],
		    XFmode, &ext_80387_constants_table[i]);
    }

  ext_80387_constants_init = 1;
}

/* Return non-zero if the constant is something that
   can be loaded with a special instruction.  */

int
standard_80387_constant_p (rtx x)
{
  machine_mode mode = GET_MODE (x);

  const REAL_VALUE_TYPE *r;

  if (!(CONST_DOUBLE_P (x) && X87_FLOAT_MODE_P (mode)))
    return -1;

  if (x == CONST0_RTX (mode))
    return 1;
  if (x == CONST1_RTX (mode))
    return 2;

  r = CONST_DOUBLE_REAL_VALUE (x);

  /* For XFmode constants, try to find a special 80387 instruction when
     optimizing for size or on those CPUs that benefit from them.  */
  if (mode == XFmode
      && (optimize_function_for_size_p (cfun) || TARGET_EXT_80387_CONSTANTS)
      && !flag_rounding_math)
    {
      int i;

      if (! ext_80387_constants_init)
	init_ext_80387_constants ();

      for (i = 0; i < 5; i++)
        if (real_identical (r, &ext_80387_constants_table[i]))
	  return i + 3;
    }

  /* Load of the constant -0.0 or -1.0 will be split as
     fldz;fchs or fld1;fchs sequence.  */
  if (real_isnegzero (r))
    return 8;
  if (real_identical (r, &dconstm1))
    return 9;

  return 0;
}

/* Return the opcode of the special instruction to be used to load
   the constant X.  */

const char *
standard_80387_constant_opcode (rtx x)
{
  switch (standard_80387_constant_p (x))
    {
    case 1:
      return "fldz";
    case 2:
      return "fld1";
    case 3:
      return "fldlg2";
    case 4:
      return "fldln2";
    case 5:
      return "fldl2e";
    case 6:
      return "fldl2t";
    case 7:
      return "fldpi";
    case 8:
    case 9:
      return "#";
    default:
      gcc_unreachable ();
    }
}

/* Return the CONST_DOUBLE representing the 80387 constant that is
   loaded by the specified special instruction.  The argument IDX
   matches the return value from standard_80387_constant_p.  */

rtx
standard_80387_constant_rtx (int idx)
{
  int i;

  if (! ext_80387_constants_init)
    init_ext_80387_constants ();

  switch (idx)
    {
    case 3:
    case 4:
    case 5:
    case 6:
    case 7:
      i = idx - 3;
      break;

    default:
      gcc_unreachable ();
    }

  return const_double_from_real_value (ext_80387_constants_table[i],
				       XFmode);
}

/* Return 1 if X is all bits 0, 2 if X is all bits 1
   and 3 if X is all bits 1 with zero extend
   in supported SSE/AVX vector mode.  */

int
standard_sse_constant_p (rtx x, machine_mode pred_mode)
{
  machine_mode mode;

  if (!TARGET_SSE)
    return 0;

  mode = GET_MODE (x);

  if (x == const0_rtx || const0_operand (x, mode))
    return 1;

  if (x == constm1_rtx
      || vector_all_ones_operand (x, mode)
      || ((GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT
	   || GET_MODE_CLASS (pred_mode) == MODE_VECTOR_FLOAT)
	  && float_vector_all_ones_operand (x, mode)))
    {
      /* VOIDmode integer constant, get mode from the predicate.  */
      if (mode == VOIDmode)
	mode = pred_mode;

      switch (GET_MODE_SIZE (mode))
	{
	case 64:
	  if (TARGET_AVX512F)
	    return 2;
	  break;
	case 32:
	  if (TARGET_AVX2)
	    return 2;
	  break;
	case 16:
	  if (TARGET_SSE2)
	    return 2;
	  break;
	case 0:
	  /* VOIDmode */
	  gcc_unreachable ();
	default:
	  break;
	}
    }

  if (vector_all_ones_zero_extend_half_operand (x, mode)
      || vector_all_ones_zero_extend_quarter_operand (x, mode))
    return 3;

  return 0;
}

/* Return the opcode of the special instruction to be used to load
   the constant operands[1] into operands[0].  */

const char *
standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
{
  machine_mode mode;
  rtx x = operands[1];

  gcc_assert (TARGET_SSE);

  mode = GET_MODE (x);

  if (x == const0_rtx || const0_operand (x, mode))
    {
      switch (get_attr_mode (insn))
	{
	case MODE_TI:
	  if (!EXT_REX_SSE_REG_P (operands[0]))
	    return "%vpxor\t%0, %d0";
	  /* FALLTHRU */
	case MODE_XI:
	case MODE_OI:
	  if (EXT_REX_SSE_REG_P (operands[0]))
	    return (TARGET_AVX512VL
		    ? "vpxord\t%x0, %x0, %x0"
		    : "vpxord\t%g0, %g0, %g0");
	  return "vpxor\t%x0, %x0, %x0";

	case MODE_V2DF:
	  if (!EXT_REX_SSE_REG_P (operands[0]))
	    return "%vxorpd\t%0, %d0";
	  /* FALLTHRU */
	case MODE_V8DF:
	case MODE_V4DF:
	  if (!EXT_REX_SSE_REG_P (operands[0]))
	    return "vxorpd\t%x0, %x0, %x0";
	  else if (TARGET_AVX512DQ)
	    return (TARGET_AVX512VL
		    ? "vxorpd\t%x0, %x0, %x0"
		    : "vxorpd\t%g0, %g0, %g0");
	  else
	    return (TARGET_AVX512VL
		    ? "vpxorq\t%x0, %x0, %x0"
		    : "vpxorq\t%g0, %g0, %g0");

	case MODE_V4SF:
	  if (!EXT_REX_SSE_REG_P (operands[0]))
	    return "%vxorps\t%0, %d0";
	  /* FALLTHRU */
	case MODE_V16SF:
	case MODE_V8SF:
	  if (!EXT_REX_SSE_REG_P (operands[0]))
	    return "vxorps\t%x0, %x0, %x0";
	  else if (TARGET_AVX512DQ)
	    return (TARGET_AVX512VL
		    ? "vxorps\t%x0, %x0, %x0"
		    : "vxorps\t%g0, %g0, %g0");
	  else
	    return (TARGET_AVX512VL
		    ? "vpxord\t%x0, %x0, %x0"
		    : "vpxord\t%g0, %g0, %g0");

	default:
	  gcc_unreachable ();
	}
    }
  else if (x == constm1_rtx
	   || vector_all_ones_operand (x, mode)
	   || (GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT
	       && float_vector_all_ones_operand (x, mode)))
    {
      enum attr_mode insn_mode = get_attr_mode (insn);
      
      switch (insn_mode)
	{
	case MODE_XI:
	case MODE_V8DF:
	case MODE_V16SF:
	  gcc_assert (TARGET_AVX512F);
	  return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";

	case MODE_OI:
	case MODE_V4DF:
	case MODE_V8SF:
	  gcc_assert (TARGET_AVX2);
	  /* FALLTHRU */
	case MODE_TI:
	case MODE_V2DF:
	case MODE_V4SF:
	  gcc_assert (TARGET_SSE2);
	  if (!EXT_REX_SSE_REG_P (operands[0]))
	    return (TARGET_AVX
		    ? "vpcmpeqd\t%0, %0, %0"
		    : "pcmpeqd\t%0, %0");
	  else if (TARGET_AVX512VL)
	    return "vpternlogd\t{$0xFF, %0, %0, %0|%0, %0, %0, 0xFF}";
	  else
	    return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";

	default:
	  gcc_unreachable ();
	}
   }
  else if (vector_all_ones_zero_extend_half_operand (x, mode))
    {
      if (GET_MODE_SIZE (mode) == 64)
	{
	  gcc_assert (TARGET_AVX512F);
	  return "vpcmpeqd\t%t0, %t0, %t0";
	}
      else if (GET_MODE_SIZE (mode) == 32)
	{
	  gcc_assert (TARGET_AVX);
	  return "vpcmpeqd\t%x0, %x0, %x0";
	}
      gcc_unreachable ();
    }
  else if (vector_all_ones_zero_extend_quarter_operand (x, mode))
    {
      gcc_assert (TARGET_AVX512F);
      return "vpcmpeqd\t%x0, %x0, %x0";
    }

  gcc_unreachable ();
}

/* Returns true if INSN can be transformed from a memory load
   to a supported FP constant load.  */

bool
ix86_standard_x87sse_constant_load_p (const rtx_insn *insn, rtx dst)
{
  rtx src = find_constant_src (insn);

  gcc_assert (REG_P (dst));

  if (src == NULL
      || (SSE_REGNO_P (REGNO (dst))
	  && standard_sse_constant_p (src, GET_MODE (dst)) != 1)
      || (STACK_REGNO_P (REGNO (dst))
	   && standard_80387_constant_p (src) < 1))
    return false;

  return true;
}

/* Predicate for pre-reload splitters with associated instructions,
   which can match any time before the split1 pass (usually combine),
   then are unconditionally split in that pass and should not be
   matched again afterwards.  */

bool
ix86_pre_reload_split (void)
{
  return (can_create_pseudo_p ()
	  && !(cfun->curr_properties & PROP_rtl_split_insns));
}

/* Return the opcode of the TYPE_SSEMOV instruction.  To move from
   or to xmm16-xmm31/ymm16-ymm31 registers, we either require
   TARGET_AVX512VL or it is a register to register move which can
   be done with zmm register move. */

static const char *
ix86_get_ssemov (rtx *operands, unsigned size,
		 enum attr_mode insn_mode, machine_mode mode)
{
  char buf[128];
  bool misaligned_p = (misaligned_operand (operands[0], mode)
		       || misaligned_operand (operands[1], mode));
  bool evex_reg_p = (size == 64
		     || EXT_REX_SSE_REG_P (operands[0])
		     || EXT_REX_SSE_REG_P (operands[1]));
  machine_mode scalar_mode;

  const char *opcode = NULL;
  enum
    {
      opcode_int,
      opcode_float,
      opcode_double
    } type = opcode_int;

  switch (insn_mode)
    {
    case MODE_V16SF:
    case MODE_V8SF:
    case MODE_V4SF:
      scalar_mode = E_SFmode;
      type = opcode_float;
      break;
    case MODE_V8DF:
    case MODE_V4DF:
    case MODE_V2DF:
      scalar_mode = E_DFmode;
      type = opcode_double;
      break;
    case MODE_XI:
    case MODE_OI:
    case MODE_TI:
      scalar_mode = GET_MODE_INNER (mode);
      break;
    default:
      gcc_unreachable ();
    }

  /* NB: To move xmm16-xmm31/ymm16-ymm31 registers without AVX512VL,
     we can only use zmm register move without memory operand.  */
  if (evex_reg_p
      && !TARGET_AVX512VL
      && GET_MODE_SIZE (mode) < 64)
    {
      /* NB: Even though ix86_hard_regno_mode_ok doesn't allow
	 xmm16-xmm31 nor ymm16-ymm31 in 128/256 bit modes when
	 AVX512VL is disabled, LRA can still generate reg to
	 reg moves with xmm16-xmm31 and ymm16-ymm31 in 128/256 bit
	 modes.  */
      if (memory_operand (operands[0], mode)
	  || memory_operand (operands[1], mode))
	gcc_unreachable ();
      size = 64;
      switch (type)
	{
	case opcode_int:
	  if (scalar_mode == E_HFmode || scalar_mode == E_BFmode)
	    opcode = (misaligned_p
		      ? (TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64")
		      : "vmovdqa64");
	  else
	    opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32";
	  break;
	case opcode_float:
	  opcode = misaligned_p ? "vmovups" : "vmovaps";
	  break;
	case opcode_double:
	  opcode = misaligned_p ? "vmovupd" : "vmovapd";
	  break;
	}
    }
  else if (SCALAR_FLOAT_MODE_P (scalar_mode))
    {
      switch (scalar_mode)
	{
	case E_HFmode:
	case E_BFmode:
	  if (evex_reg_p)
	    opcode = (misaligned_p
		      ? (TARGET_AVX512BW
			 ? "vmovdqu16"
			 : "vmovdqu64")
		      : "vmovdqa64");
	  else
	    opcode = (misaligned_p
		      ? (TARGET_AVX512BW
			 ? "vmovdqu16"
			 : "%vmovdqu")
		      : "%vmovdqa");
	  break;
	case E_SFmode:
	  opcode = misaligned_p ? "%vmovups" : "%vmovaps";
	  break;
	case E_DFmode:
	  opcode = misaligned_p ? "%vmovupd" : "%vmovapd";
	  break;
	case E_TFmode:
	  if (evex_reg_p)
	    opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
	  else
	    opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
	  break;
	default:
	  gcc_unreachable ();
	}
    }
  else if (SCALAR_INT_MODE_P (scalar_mode))
    {
      switch (scalar_mode)
	{
	case E_QImode:
	  if (evex_reg_p)
	    opcode = (misaligned_p
		      ? (TARGET_AVX512BW
			 ? "vmovdqu8"
			 : "vmovdqu64")
		      : "vmovdqa64");
	  else
	    opcode = (misaligned_p
		      ? (TARGET_AVX512BW
			 ? "vmovdqu8"
			 : "%vmovdqu")
		      : "%vmovdqa");
	  break;
	case E_HImode:
	  if (evex_reg_p)
	    opcode = (misaligned_p
		      ? (TARGET_AVX512BW
			 ? "vmovdqu16"
			 : "vmovdqu64")
		      : "vmovdqa64");
	  else
	    opcode = (misaligned_p
		      ? (TARGET_AVX512BW
			 ? "vmovdqu16"
			 : "%vmovdqu")
		      : "%vmovdqa");
	  break;
	case E_SImode:
	  if (evex_reg_p)
	    opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32";
	  else
	    opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
	  break;
	case E_DImode:
	case E_TImode:
	case E_OImode:
	  if (evex_reg_p)
	    opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
	  else
	    opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
	  break;
	case E_XImode:
	  opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
	  break;
	default:
	  gcc_unreachable ();
	}
    }
  else
    gcc_unreachable ();

  switch (size)
    {
    case 64:
      snprintf (buf, sizeof (buf), "%s\t{%%g1, %%g0|%%g0, %%g1}",
		opcode);
      break;
    case 32:
      snprintf (buf, sizeof (buf), "%s\t{%%t1, %%t0|%%t0, %%t1}",
		opcode);
      break;
    case 16:
      snprintf (buf, sizeof (buf), "%s\t{%%x1, %%x0|%%x0, %%x1}",
		opcode);
      break;
    default:
      gcc_unreachable ();
    }
  output_asm_insn (buf, operands);
  return "";
}

/* Return the template of the TYPE_SSEMOV instruction to move
   operands[1] into operands[0].  */

const char *
ix86_output_ssemov (rtx_insn *insn, rtx *operands)
{
  machine_mode mode = GET_MODE (operands[0]);
  if (get_attr_type (insn) != TYPE_SSEMOV
      || mode != GET_MODE (operands[1]))
    gcc_unreachable ();

  enum attr_mode insn_mode = get_attr_mode (insn);

  switch (insn_mode)
    {
    case MODE_XI:
    case MODE_V8DF:
    case MODE_V16SF:
      return ix86_get_ssemov (operands, 64, insn_mode, mode);

    case MODE_OI:
    case MODE_V4DF:
    case MODE_V8SF:
      return ix86_get_ssemov (operands, 32, insn_mode, mode);

    case MODE_TI:
    case MODE_V2DF:
    case MODE_V4SF:
      return ix86_get_ssemov (operands, 16, insn_mode, mode);

    case MODE_DI:
      /* Handle broken assemblers that require movd instead of movq. */
      if (GENERAL_REG_P (operands[0]))
	{
	  if (HAVE_AS_IX86_INTERUNIT_MOVQ)
	    return "%vmovq\t{%1, %q0|%q0, %1}";
	  else
	    return "%vmovd\t{%1, %q0|%q0, %1}";
	}
      else if (GENERAL_REG_P (operands[1]))
	{
	  if (HAVE_AS_IX86_INTERUNIT_MOVQ)
	    return "%vmovq\t{%q1, %0|%0, %q1}";
	  else
	    return "%vmovd\t{%q1, %0|%0, %q1}";
	}
      else
	return "%vmovq\t{%1, %0|%0, %1}";

    case MODE_SI:
      if (GENERAL_REG_P (operands[0]))
	return "%vmovd\t{%1, %k0|%k0, %1}";
      else if (GENERAL_REG_P (operands[1]))
	return "%vmovd\t{%k1, %0|%0, %k1}";
      else
	return "%vmovd\t{%1, %0|%0, %1}";

    case MODE_HI:
      if (GENERAL_REG_P (operands[0]))
	return "vmovw\t{%1, %k0|%k0, %1}";
      else if (GENERAL_REG_P (operands[1]))
	return "vmovw\t{%k1, %0|%0, %k1}";
      else
	return "vmovw\t{%1, %0|%0, %1}";

    case MODE_DF:
      if (TARGET_AVX && REG_P (operands[0]) && REG_P (operands[1]))
	return "vmovsd\t{%d1, %0|%0, %d1}";
      else
	return "%vmovsd\t{%1, %0|%0, %1}";

    case MODE_SF:
      if (TARGET_AVX && REG_P (operands[0]) && REG_P (operands[1]))
	return "vmovss\t{%d1, %0|%0, %d1}";
      else
	return "%vmovss\t{%1, %0|%0, %1}";

    case MODE_HF:
    case MODE_BF:
      if (REG_P (operands[0]) && REG_P (operands[1]))
	return "vmovsh\t{%d1, %0|%0, %d1}";
      else
	return "vmovsh\t{%1, %0|%0, %1}";

    case MODE_V1DF:
      gcc_assert (!TARGET_AVX);
      return "movlpd\t{%1, %0|%0, %1}";

    case MODE_V2SF:
      if (TARGET_AVX && REG_P (operands[0]))
	return "vmovlps\t{%1, %d0|%d0, %1}";
      else
	return "%vmovlps\t{%1, %0|%0, %1}";

    default:
      gcc_unreachable ();
    }
}

/* Returns true if OP contains a symbol reference */

bool
symbolic_reference_mentioned_p (rtx op)
{
  const char *fmt;
  int i;

  if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF)
    return true;

  fmt = GET_RTX_FORMAT (GET_CODE (op));
  for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--)
    {
      if (fmt[i] == 'E')
	{
	  int j;

	  for (j = XVECLEN (op, i) - 1; j >= 0; j--)
	    if (symbolic_reference_mentioned_p (XVECEXP (op, i, j)))
	      return true;
	}

      else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i)))
	return true;
    }

  return false;
}

/* Return true if it is appropriate to emit `ret' instructions in the
   body of a function.  Do this only if the epilogue is simple, needing a
   couple of insns.  Prior to reloading, we can't tell how many registers
   must be saved, so return false then.  Return false if there is no frame
   marker to de-allocate.  */

bool
ix86_can_use_return_insn_p (void)
{
  if (ix86_function_ms_hook_prologue (current_function_decl))
    return false;

  if (ix86_function_naked (current_function_decl))
    return false;

  /* Don't use `ret' instruction in interrupt handler.  */
  if (! reload_completed
      || frame_pointer_needed
      || cfun->machine->func_type != TYPE_NORMAL)
    return 0;

  /* Don't allow more than 32k pop, since that's all we can do
     with one instruction.  */
  if (crtl->args.pops_args && crtl->args.size >= 32768)
    return 0;

  struct ix86_frame &frame = cfun->machine->frame;
  return (frame.stack_pointer_offset == UNITS_PER_WORD
	  && (frame.nregs + frame.nsseregs) == 0);
}

/* Return stack frame size.  get_frame_size () returns used stack slots
   during compilation, which may be optimized out later.  If stack frame
   is needed, stack_frame_required should be true.  */

static HOST_WIDE_INT
ix86_get_frame_size (void)
{
  if (cfun->machine->stack_frame_required)
    return get_frame_size ();
  else
    return 0;
}

/* Value should be nonzero if functions must have frame pointers.
   Zero means the frame pointer need not be set up (and parms may
   be accessed via the stack pointer) in functions that seem suitable.  */

static bool
ix86_frame_pointer_required (void)
{
  /* If we accessed previous frames, then the generated code expects
     to be able to access the saved ebp value in our frame.  */
  if (cfun->machine->accesses_prev_frame)
    return true;

  /* Several x86 os'es need a frame pointer for other reasons,
     usually pertaining to setjmp.  */
  if (SUBTARGET_FRAME_POINTER_REQUIRED)
    return true;

  /* For older 32-bit runtimes setjmp requires valid frame-pointer.  */
  if (TARGET_32BIT_MS_ABI && cfun->calls_setjmp)
    return true;

  /* Win64 SEH, very large frames need a frame-pointer as maximum stack
     allocation is 4GB.  */
  if (TARGET_64BIT_MS_ABI && ix86_get_frame_size () > SEH_MAX_FRAME_SIZE)
    return true;

  /* SSE saves require frame-pointer when stack is misaligned.  */
  if (TARGET_64BIT_MS_ABI && ix86_incoming_stack_boundary < 128)
    return true;
  
  /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
     turns off the frame pointer by default.  Turn it back on now if
     we've not got a leaf function.  */
  if (TARGET_OMIT_LEAF_FRAME_POINTER
      && (!crtl->is_leaf
	  || ix86_current_function_calls_tls_descriptor))
    return true;

  /* Several versions of mcount for the x86 assumes that there is a
     frame, so we cannot allow profiling without a frame pointer.  */
  if (crtl->profile && !flag_fentry)
    return true;

  return false;
}

/* Record that the current function accesses previous call frames.  */

void
ix86_setup_frame_addresses (void)
{
  cfun->machine->accesses_prev_frame = 1;
}

#ifndef USE_HIDDEN_LINKONCE
# if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
#  define USE_HIDDEN_LINKONCE 1
# else
#  define USE_HIDDEN_LINKONCE 0
# endif
#endif

/* Label count for call and return thunks.  It is used to make unique
   labels in call and return thunks.  */
static int indirectlabelno;

/* True if call thunk function is needed.  */
static bool indirect_thunk_needed = false;

/* Bit masks of integer registers, which contain branch target, used
   by call thunk functions.  */
static HARD_REG_SET indirect_thunks_used;

/* True if return thunk function is needed.  */
static bool indirect_return_needed = false;

/* True if return thunk function via CX is needed.  */
static bool indirect_return_via_cx;

#ifndef INDIRECT_LABEL
# define INDIRECT_LABEL "LIND"
#endif

/* Indicate what prefix is needed for an indirect branch.  */
enum indirect_thunk_prefix
{
  indirect_thunk_prefix_none,
  indirect_thunk_prefix_nt
};

/* Return the prefix needed for an indirect branch INSN.  */

enum indirect_thunk_prefix
indirect_thunk_need_prefix (rtx_insn *insn)
{
  enum indirect_thunk_prefix need_prefix;
  if ((cfun->machine->indirect_branch_type
	    == indirect_branch_thunk_extern)
	   && ix86_notrack_prefixed_insn_p (insn))
    {
      /* NOTRACK prefix is only used with external thunk so that it
	 can be properly updated to support CET at run-time.  */
      need_prefix = indirect_thunk_prefix_nt;
    }
  else
    need_prefix = indirect_thunk_prefix_none;
  return need_prefix;
}

/* Fills in the label name that should be used for the indirect thunk.  */

static void
indirect_thunk_name (char name[32], unsigned int regno,
		     enum indirect_thunk_prefix need_prefix,
		     bool ret_p)
{
  if (regno != INVALID_REGNUM && regno != CX_REG && ret_p)
    gcc_unreachable ();

  if (USE_HIDDEN_LINKONCE)
    {
      const char *prefix;

      if (need_prefix == indirect_thunk_prefix_nt
	  && regno != INVALID_REGNUM)
	{
	  /* NOTRACK prefix is only used with external thunk via
	     register so that NOTRACK prefix can be added to indirect
	     branch via register to support CET at run-time.  */
	  prefix = "_nt";
	}
      else
	prefix = "";

      const char *ret = ret_p ? "return" : "indirect";

      if (regno != INVALID_REGNUM)
	{
	  const char *reg_prefix;
	  if (LEGACY_INT_REGNO_P (regno))
	    reg_prefix = TARGET_64BIT ? "r" : "e";
	  else
	    reg_prefix = "";
	  sprintf (name, "__x86_%s_thunk%s_%s%s",
		   ret, prefix, reg_prefix, reg_names[regno]);
	}
      else
	sprintf (name, "__x86_%s_thunk%s", ret, prefix);
    }
  else
    {
      if (regno != INVALID_REGNUM)
	ASM_GENERATE_INTERNAL_LABEL (name, "LITR", regno);
      else
	{
	  if (ret_p)
	    ASM_GENERATE_INTERNAL_LABEL (name, "LRT", 0);
	  else
	    ASM_GENERATE_INTERNAL_LABEL (name, "LIT", 0);
	}
    }
}

/* Output a call and return thunk for indirect branch.  If REGNO != -1,
   the function address is in REGNO and the call and return thunk looks like:

	call	L2
   L1:
	pause
	lfence
	jmp	L1
   L2:
	mov	%REG, (%sp)
	ret

   Otherwise, the function address is on the top of stack and the
   call and return thunk looks like:

	call L2
  L1:
	pause
	lfence
	jmp L1
  L2:
	lea WORD_SIZE(%sp), %sp
	ret
 */

static void
output_indirect_thunk (unsigned int regno)
{
  char indirectlabel1[32];
  char indirectlabel2[32];

  ASM_GENERATE_INTERNAL_LABEL (indirectlabel1, INDIRECT_LABEL,
			       indirectlabelno++);
  ASM_GENERATE_INTERNAL_LABEL (indirectlabel2, INDIRECT_LABEL,
			       indirectlabelno++);

  /* Call */
  fputs ("\tcall\t", asm_out_file);
  assemble_name_raw (asm_out_file, indirectlabel2);
  fputc ('\n', asm_out_file);

  ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);

  /* AMD and Intel CPUs prefer each a different instruction as loop filler.
     Usage of both pause + lfence is compromise solution.  */
  fprintf (asm_out_file, "\tpause\n\tlfence\n");

  /* Jump.  */
  fputs ("\tjmp\t", asm_out_file);
  assemble_name_raw (asm_out_file, indirectlabel1);
  fputc ('\n', asm_out_file);

  ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);

  /* The above call insn pushed a word to stack.  Adjust CFI info.  */
  if (flag_asynchronous_unwind_tables && dwarf2out_do_frame ())
    {
      if (! dwarf2out_do_cfi_asm ())
	{
	  dw_cfi_ref xcfi = ggc_cleared_alloc<dw_cfi_node> ();
	  xcfi->dw_cfi_opc = DW_CFA_advance_loc4;
	  xcfi->dw_cfi_oprnd1.dw_cfi_addr = ggc_strdup (indirectlabel2);
	  vec_safe_push (cfun->fde->dw_fde_cfi, xcfi);
	}
      dw_cfi_ref xcfi = ggc_cleared_alloc<dw_cfi_node> ();
      xcfi->dw_cfi_opc = DW_CFA_def_cfa_offset;
      xcfi->dw_cfi_oprnd1.dw_cfi_offset = 2 * UNITS_PER_WORD;
      vec_safe_push (cfun->fde->dw_fde_cfi, xcfi);
      dwarf2out_emit_cfi (xcfi);
    }

  if (regno != INVALID_REGNUM)
    {
      /* MOV.  */
      rtx xops[2];
      xops[0] = gen_rtx_MEM (word_mode, stack_pointer_rtx);
      xops[1] = gen_rtx_REG (word_mode, regno);
      output_asm_insn ("mov\t{%1, %0|%0, %1}", xops);
    }
  else
    {
      /* LEA.  */
      rtx xops[2];
      xops[0] = stack_pointer_rtx;
      xops[1] = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
      output_asm_insn ("lea\t{%E1, %0|%0, %E1}", xops);
    }

  fputs ("\tret\n", asm_out_file);
  if ((ix86_harden_sls & harden_sls_return))
    fputs ("\tint3\n", asm_out_file);
}

/* Output a funtion with a call and return thunk for indirect branch.
   If REGNO != INVALID_REGNUM, the function address is in REGNO.
   Otherwise, the function address is on the top of stack.  Thunk is
   used for function return if RET_P is true.  */

static void
output_indirect_thunk_function (enum indirect_thunk_prefix need_prefix,
				unsigned int regno, bool ret_p)
{
  char name[32];
  tree decl;

  /* Create __x86_indirect_thunk.  */
  indirect_thunk_name (name, regno, need_prefix, ret_p);
  decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
		     get_identifier (name),
		     build_function_type_list (void_type_node, NULL_TREE));
  DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
				   NULL_TREE, void_type_node);
  TREE_PUBLIC (decl) = 1;
  TREE_STATIC (decl) = 1;
  DECL_IGNORED_P (decl) = 1;

#if TARGET_MACHO
  if (TARGET_MACHO)
    {
      switch_to_section (darwin_sections[picbase_thunk_section]);
      fputs ("\t.weak_definition\t", asm_out_file);
      assemble_name (asm_out_file, name);
      fputs ("\n\t.private_extern\t", asm_out_file);
      assemble_name (asm_out_file, name);
      putc ('\n', asm_out_file);
      ASM_OUTPUT_LABEL (asm_out_file, name);
      DECL_WEAK (decl) = 1;
    }
  else
#endif
    if (USE_HIDDEN_LINKONCE)
      {
	cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));

	targetm.asm_out.unique_section (decl, 0);
	switch_to_section (get_named_section (decl, NULL, 0));

	targetm.asm_out.globalize_label (asm_out_file, name);
	fputs ("\t.hidden\t", asm_out_file);
	assemble_name (asm_out_file, name);
	putc ('\n', asm_out_file);
	ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
      }
    else
      {
	switch_to_section (text_section);
	ASM_OUTPUT_LABEL (asm_out_file, name);
      }

  DECL_INITIAL (decl) = make_node (BLOCK);
  current_function_decl = decl;
  allocate_struct_function (decl, false);
  init_function_start (decl);
  /* We're about to hide the function body from callees of final_* by
     emitting it directly; tell them we're a thunk, if they care.  */
  cfun->is_thunk = true;
  first_function_block_is_cold = false;
  /* Make sure unwind info is emitted for the thunk if needed.  */
  final_start_function (emit_barrier (), asm_out_file, 1);

  output_indirect_thunk (regno);

  final_end_function ();
  init_insn_lengths ();
  free_after_compilation (cfun);
  set_cfun (NULL);
  current_function_decl = NULL;
}

static int pic_labels_used;

/* Fills in the label name that should be used for a pc thunk for
   the given register.  */

static void
get_pc_thunk_name (char name[32], unsigned int regno)
{
  gcc_assert (!TARGET_64BIT);

  if (USE_HIDDEN_LINKONCE)
    sprintf (name, "__x86.get_pc_thunk.%s", reg_names[regno]);
  else
    ASM_GENERATE_INTERNAL_LABEL (name, "LPR", regno);
}


/* This function generates code for -fpic that loads %ebx with
   the return address of the caller and then returns.  */

static void
ix86_code_end (void)
{
  rtx xops[2];
  unsigned int regno;

  if (indirect_return_needed)
    output_indirect_thunk_function (indirect_thunk_prefix_none,
				    INVALID_REGNUM, true);
  if (indirect_return_via_cx)
    output_indirect_thunk_function (indirect_thunk_prefix_none,
				    CX_REG, true);
  if (indirect_thunk_needed)
    output_indirect_thunk_function (indirect_thunk_prefix_none,
				    INVALID_REGNUM, false);

  for (regno = FIRST_REX_INT_REG; regno <= LAST_REX_INT_REG; regno++)
    {
      if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
	output_indirect_thunk_function (indirect_thunk_prefix_none,
					regno, false);
    }

  for (regno = FIRST_REX2_INT_REG; regno <= LAST_REX2_INT_REG; regno++)
    {
      if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
	output_indirect_thunk_function (indirect_thunk_prefix_none,
					regno, false);
    }

  for (regno = FIRST_INT_REG; regno <= LAST_INT_REG; regno++)
    {
      char name[32];
      tree decl;

      if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
	output_indirect_thunk_function (indirect_thunk_prefix_none,
					regno, false);

      if (!(pic_labels_used & (1 << regno)))
	continue;

      get_pc_thunk_name (name, regno);

      decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
			 get_identifier (name),
			 build_function_type_list (void_type_node, NULL_TREE));
      DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
				       NULL_TREE, void_type_node);
      TREE_PUBLIC (decl) = 1;
      TREE_STATIC (decl) = 1;
      DECL_IGNORED_P (decl) = 1;

#if TARGET_MACHO
      if (TARGET_MACHO)
	{
	  switch_to_section (darwin_sections[picbase_thunk_section]);
	  fputs ("\t.weak_definition\t", asm_out_file);
	  assemble_name (asm_out_file, name);
	  fputs ("\n\t.private_extern\t", asm_out_file);
	  assemble_name (asm_out_file, name);
	  putc ('\n', asm_out_file);
	  ASM_OUTPUT_LABEL (asm_out_file, name);
	  DECL_WEAK (decl) = 1;
	}
      else
#endif
      if (USE_HIDDEN_LINKONCE)
	{
	  cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));

	  targetm.asm_out.unique_section (decl, 0);
	  switch_to_section (get_named_section (decl, NULL, 0));

	  targetm.asm_out.globalize_label (asm_out_file, name);
	  fputs ("\t.hidden\t", asm_out_file);
	  assemble_name (asm_out_file, name);
	  putc ('\n', asm_out_file);
	  ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
	}
      else
	{
	  switch_to_section (text_section);
	  ASM_OUTPUT_LABEL (asm_out_file, name);
	}

      DECL_INITIAL (decl) = make_node (BLOCK);
      current_function_decl = decl;
      allocate_struct_function (decl, false);
      init_function_start (decl);
      /* We're about to hide the function body from callees of final_* by
	 emitting it directly; tell them we're a thunk, if they care.  */
      cfun->is_thunk = true;
      first_function_block_is_cold = false;
      /* Make sure unwind info is emitted for the thunk if needed.  */
      final_start_function (emit_barrier (), asm_out_file, 1);

      /* Pad stack IP move with 4 instructions (two NOPs count
	 as one instruction).  */
      if (TARGET_PAD_SHORT_FUNCTION)
	{
	  int i = 8;

	  while (i--)
	    fputs ("\tnop\n", asm_out_file);
	}

      xops[0] = gen_rtx_REG (Pmode, regno);
      xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
      output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
      fputs ("\tret\n", asm_out_file);
      final_end_function ();
      init_insn_lengths ();
      free_after_compilation (cfun);
      set_cfun (NULL);
      current_function_decl = NULL;
    }

  if (flag_split_stack)
    file_end_indicate_split_stack ();
}

/* Emit code for the SET_GOT patterns.  */

const char *
output_set_got (rtx dest, rtx label)
{
  rtx xops[3];

  xops[0] = dest;

  if (TARGET_VXWORKS_RTP && flag_pic)
    {
      /* Load (*VXWORKS_GOTT_BASE) into the PIC register.  */
      xops[2] = gen_rtx_MEM (Pmode,
			     gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_BASE));
      output_asm_insn ("mov{l}\t{%2, %0|%0, %2}", xops);

      /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
	 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
	 an unadorned address.  */
      xops[2] = gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_INDEX);
      SYMBOL_REF_FLAGS (xops[2]) |= SYMBOL_FLAG_LOCAL;
      output_asm_insn ("mov{l}\t{%P2(%0), %0|%0, DWORD PTR %P2[%0]}", xops);
      return "";
    }

  xops[1] = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);

  if (flag_pic)
    {
      char name[32];
      get_pc_thunk_name (name, REGNO (dest));
      pic_labels_used |= 1 << REGNO (dest);

      xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
      xops[2] = gen_rtx_MEM (QImode, xops[2]);
      output_asm_insn ("%!call\t%X2", xops);

#if TARGET_MACHO
      /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
         This is what will be referenced by the Mach-O PIC subsystem.  */
      if (machopic_should_output_picbase_label () || !label)
	ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);

      /* When we are restoring the pic base at the site of a nonlocal label,
         and we decided to emit the pic base above, we will still output a
         local label used for calculating the correction offset (even though
         the offset will be 0 in that case).  */
      if (label)
        targetm.asm_out.internal_label (asm_out_file, "L",
					   CODE_LABEL_NUMBER (label));
#endif
    }
  else
    {
      if (TARGET_MACHO)
	/* We don't need a pic base, we're not producing pic.  */
	gcc_unreachable ();

      xops[2] = gen_rtx_LABEL_REF (Pmode, label ? label : gen_label_rtx ());
      output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops);
      targetm.asm_out.internal_label (asm_out_file, "L",
				      CODE_LABEL_NUMBER (XEXP (xops[2], 0)));
    }

  if (!TARGET_MACHO)
    output_asm_insn ("add%z0\t{%1, %0|%0, %1}", xops);

  return "";
}

/* Generate an "push" pattern for input ARG.  */

rtx
gen_push (rtx arg)
{
  struct machine_function *m = cfun->machine;

  if (m->fs.cfa_reg == stack_pointer_rtx)
    m->fs.cfa_offset += UNITS_PER_WORD;
  m->fs.sp_offset += UNITS_PER_WORD;

  if (REG_P (arg) && GET_MODE (arg) != word_mode)
    arg = gen_rtx_REG (word_mode, REGNO (arg));

  return gen_rtx_SET (gen_rtx_MEM (word_mode,
				   gen_rtx_PRE_DEC (Pmode,
						    stack_pointer_rtx)),
		      arg);
}

/* Generate an "pop" pattern for input ARG.  */

rtx
gen_pop (rtx arg)
{
  if (REG_P (arg) && GET_MODE (arg) != word_mode)
    arg = gen_rtx_REG (word_mode, REGNO (arg));

  return gen_rtx_SET (arg,
		      gen_rtx_MEM (word_mode,
				   gen_rtx_POST_INC (Pmode,
						     stack_pointer_rtx)));
}

/* Return >= 0 if there is an unused call-clobbered register available
   for the entire function.  */

static unsigned int
ix86_select_alt_pic_regnum (void)
{
  if (ix86_use_pseudo_pic_reg ())
    return INVALID_REGNUM;

  if (crtl->is_leaf
      && !crtl->profile
      && !ix86_current_function_calls_tls_descriptor)
    {
      int i, drap;
      /* Can't use the same register for both PIC and DRAP.  */
      if (crtl->drap_reg)
	drap = REGNO (crtl->drap_reg);
      else
	drap = -1;
      for (i = 2; i >= 0; --i)
        if (i != drap && !df_regs_ever_live_p (i))
	  return i;
    }

  return INVALID_REGNUM;
}

/* Return true if REGNO is used by the epilogue.  */

bool
ix86_epilogue_uses (int regno)
{
  /* If there are no caller-saved registers, we preserve all registers,
     except for MMX and x87 registers which aren't supported when saving
     and restoring registers.  Don't explicitly save SP register since
     it is always preserved.  */
  return (epilogue_completed
	  && cfun->machine->no_caller_saved_registers
	  && !fixed_regs[regno]
	  && !STACK_REGNO_P (regno)
	  && !MMX_REGNO_P (regno));
}

/* Return nonzero if register REGNO can be used as a scratch register
   in peephole2.  */

static bool
ix86_hard_regno_scratch_ok (unsigned int regno)
{
  /* If there are no caller-saved registers, we can't use any register
     as a scratch register after epilogue and use REGNO as scratch
     register only if it has been used before to avoid saving and
     restoring it.  */
  return (!cfun->machine->no_caller_saved_registers
	  || (!epilogue_completed
	      && df_regs_ever_live_p (regno)));
}

/* Return TRUE if we need to save REGNO.  */

bool
ix86_save_reg (unsigned int regno, bool maybe_eh_return, bool ignore_outlined)
{
  /* If there are no caller-saved registers, we preserve all registers,
     except for MMX and x87 registers which aren't supported when saving
     and restoring registers.  Don't explicitly save SP register since
     it is always preserved.  */
  if (cfun->machine->no_caller_saved_registers)
    {
      /* Don't preserve registers used for function return value.  */
      rtx reg = crtl->return_rtx;
      if (reg)
	{
	  unsigned int i = REGNO (reg);
	  unsigned int nregs = REG_NREGS (reg);
	  while (nregs-- > 0)
	    if ((i + nregs) == regno)
	      return false;
	}

      return (df_regs_ever_live_p (regno)
	      && !fixed_regs[regno]
	      && !STACK_REGNO_P (regno)
	      && !MMX_REGNO_P (regno)
	      && (regno != HARD_FRAME_POINTER_REGNUM
		  || !frame_pointer_needed));
    }

  if (regno == REAL_PIC_OFFSET_TABLE_REGNUM
      && pic_offset_table_rtx)
    {
      if (ix86_use_pseudo_pic_reg ())
	{
	  /* REAL_PIC_OFFSET_TABLE_REGNUM used by call to
	  _mcount in prologue.  */
	  if (!TARGET_64BIT && flag_pic && crtl->profile)
	    return true;
	}
      else if (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
	       || crtl->profile
	       || crtl->calls_eh_return
	       || crtl->uses_const_pool
	       || cfun->has_nonlocal_label)
        return ix86_select_alt_pic_regnum () == INVALID_REGNUM;
    }

  if (crtl->calls_eh_return && maybe_eh_return)
    {
      unsigned i;
      for (i = 0; ; i++)
	{
	  unsigned test = EH_RETURN_DATA_REGNO (i);
	  if (test == INVALID_REGNUM)
	    break;
	  if (test == regno)
	    return true;
	}
    }

  if (ignore_outlined && cfun->machine->call_ms2sysv)
    {
      unsigned count = cfun->machine->call_ms2sysv_extra_regs
		       + xlogue_layout::MIN_REGS;
      if (xlogue_layout::is_stub_managed_reg (regno, count))
	return false;
    }

  if (crtl->drap_reg
      && regno == REGNO (crtl->drap_reg)
      && !cfun->machine->no_drap_save_restore)
    return true;

  return (df_regs_ever_live_p (regno)
	  && !call_used_or_fixed_reg_p (regno)
	  && (regno != HARD_FRAME_POINTER_REGNUM || !frame_pointer_needed));
}

/* Return number of saved general prupose registers.  */

static int
ix86_nsaved_regs (void)
{
  int nregs = 0;
  int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
      nregs ++;
  return nregs;
}

/* Return number of saved SSE registers.  */

static int
ix86_nsaved_sseregs (void)
{
  int nregs = 0;
  int regno;

  if (!TARGET_64BIT_MS_ABI)
    return 0;
  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true, true))
      nregs ++;
  return nregs;
}

/* Given FROM and TO register numbers, say whether this elimination is
   allowed.  If stack alignment is needed, we can only replace argument
   pointer with hard frame pointer, or replace frame pointer with stack
   pointer.  Otherwise, frame pointer elimination is automatically
   handled and all other eliminations are valid.  */

static bool
ix86_can_eliminate (const int from, const int to)
{
  if (stack_realign_fp)
    return ((from == ARG_POINTER_REGNUM
	     && to == HARD_FRAME_POINTER_REGNUM)
	    || (from == FRAME_POINTER_REGNUM
		&& to == STACK_POINTER_REGNUM));
  else
    return to == STACK_POINTER_REGNUM ? !frame_pointer_needed : true;
}

/* Return the offset between two registers, one to be eliminated, and the other
   its replacement, at the start of a routine.  */

HOST_WIDE_INT
ix86_initial_elimination_offset (int from, int to)
{
  struct ix86_frame &frame = cfun->machine->frame;

  if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM)
    return frame.hard_frame_pointer_offset;
  else if (from == FRAME_POINTER_REGNUM
	   && to == HARD_FRAME_POINTER_REGNUM)
    return frame.hard_frame_pointer_offset - frame.frame_pointer_offset;
  else
    {
      gcc_assert (to == STACK_POINTER_REGNUM);

      if (from == ARG_POINTER_REGNUM)
	return frame.stack_pointer_offset;

      gcc_assert (from == FRAME_POINTER_REGNUM);
      return frame.stack_pointer_offset - frame.frame_pointer_offset;
    }
}

/* Emits a warning for unsupported msabi to sysv pro/epilogues.  */
void
warn_once_call_ms2sysv_xlogues (const char *feature)
{
  static bool warned_once = false;
  if (!warned_once)
    {
      warning (0, "%<-mcall-ms2sysv-xlogues%> is not compatible with %s",
	       feature);
      warned_once = true;
    }
}

/* Return the probing interval for -fstack-clash-protection.  */

static HOST_WIDE_INT
get_probe_interval (void)
{
  if (flag_stack_clash_protection)
    return (HOST_WIDE_INT_1U
	    << param_stack_clash_protection_probe_interval);
  else
    return (HOST_WIDE_INT_1U << STACK_CHECK_PROBE_INTERVAL_EXP);
}

/* When using -fsplit-stack, the allocation routines set a field in
   the TCB to the bottom of the stack plus this much space, measured
   in bytes.  */

#define SPLIT_STACK_AVAILABLE 256

/* Fill structure ix86_frame about frame of currently computed function.  */

static void
ix86_compute_frame_layout (void)
{
  struct ix86_frame *frame = &cfun->machine->frame;
  struct machine_function *m = cfun->machine;
  unsigned HOST_WIDE_INT stack_alignment_needed;
  HOST_WIDE_INT offset;
  unsigned HOST_WIDE_INT preferred_alignment;
  HOST_WIDE_INT size = ix86_get_frame_size ();
  HOST_WIDE_INT to_allocate;

  /* m->call_ms2sysv is initially enabled in ix86_expand_call for all 64-bit
   * ms_abi functions that call a sysv function.  We now need to prune away
   * cases where it should be disabled.  */
  if (TARGET_64BIT && m->call_ms2sysv)
    {
      gcc_assert (TARGET_64BIT_MS_ABI);
      gcc_assert (TARGET_CALL_MS2SYSV_XLOGUES);
      gcc_assert (!TARGET_SEH);
      gcc_assert (TARGET_SSE);
      gcc_assert (!ix86_using_red_zone ());

      if (crtl->calls_eh_return)
	{
	  gcc_assert (!reload_completed);
	  m->call_ms2sysv = false;
	  warn_once_call_ms2sysv_xlogues ("__builtin_eh_return");
	}

      else if (ix86_static_chain_on_stack)
	{
	  gcc_assert (!reload_completed);
	  m->call_ms2sysv = false;
	  warn_once_call_ms2sysv_xlogues ("static call chains");
	}

      /* Finally, compute which registers the stub will manage.  */
      else
	{
	  unsigned count = xlogue_layout::count_stub_managed_regs ();
	  m->call_ms2sysv_extra_regs = count - xlogue_layout::MIN_REGS;
	  m->call_ms2sysv_pad_in = 0;
	}
    }

  frame->nregs = ix86_nsaved_regs ();
  frame->nsseregs = ix86_nsaved_sseregs ();

  /* 64-bit MS ABI seem to require stack alignment to be always 16,
     except for function prologues, leaf functions and when the defult
     incoming stack boundary is overriden at command line or via
     force_align_arg_pointer attribute.

     Darwin's ABI specifies 128b alignment for both 32 and  64 bit variants
     at call sites, including profile function calls.
 */
  if (((TARGET_64BIT_MS_ABI || TARGET_MACHO)
        && crtl->preferred_stack_boundary < 128)
      && (!crtl->is_leaf || cfun->calls_alloca != 0
	  || ix86_current_function_calls_tls_descriptor
	  || (TARGET_MACHO && crtl->profile)
	  || ix86_incoming_stack_boundary < 128))
    {
      crtl->preferred_stack_boundary = 128;
      crtl->stack_alignment_needed = 128;
    }

  stack_alignment_needed = crtl->stack_alignment_needed / BITS_PER_UNIT;
  preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT;

  gcc_assert (!size || stack_alignment_needed);
  gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
  gcc_assert (preferred_alignment <= stack_alignment_needed);

  /* The only ABI saving SSE regs should be 64-bit ms_abi.  */
  gcc_assert (TARGET_64BIT || !frame->nsseregs);
  if (TARGET_64BIT && m->call_ms2sysv)
    {
      gcc_assert (stack_alignment_needed >= 16);
      gcc_assert (!frame->nsseregs);
    }

  /* For SEH we have to limit the amount of code movement into the prologue.
     At present we do this via a BLOCKAGE, at which point there's very little
     scheduling that can be done, which means that there's very little point
     in doing anything except PUSHs.  */
  if (TARGET_SEH)
    m->use_fast_prologue_epilogue = false;
  else if (!optimize_bb_for_size_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
    {
      int count = frame->nregs;
      struct cgraph_node *node = cgraph_node::get (current_function_decl);

      /* The fast prologue uses move instead of push to save registers.  This
         is significantly longer, but also executes faster as modern hardware
         can execute the moves in parallel, but can't do that for push/pop.

	 Be careful about choosing what prologue to emit:  When function takes
	 many instructions to execute we may use slow version as well as in
	 case function is known to be outside hot spot (this is known with
	 feedback only).  Weight the size of function by number of registers
	 to save as it is cheap to use one or two push instructions but very
	 slow to use many of them.

	 Calling this hook multiple times with the same frame requirements
	 must produce the same layout, since the RA might otherwise be
	 unable to reach a fixed point or might fail its final sanity checks.
	 This means that once we've assumed that a function does or doesn't
	 have a particular size, we have to stick to that assumption
	 regardless of how the function has changed since.  */
      if (count)
	count = (count - 1) * FAST_PROLOGUE_INSN_COUNT;
      if (node->frequency < NODE_FREQUENCY_NORMAL
	  || (flag_branch_probabilities
	      && node->frequency < NODE_FREQUENCY_HOT))
	m->use_fast_prologue_epilogue = false;
      else
	{
	  if (count != frame->expensive_count)
	    {
	      frame->expensive_count = count;
	      frame->expensive_p = expensive_function_p (count);
	    }
	  m->use_fast_prologue_epilogue = !frame->expensive_p;
	}
    }

  frame->save_regs_using_mov
    = TARGET_PROLOGUE_USING_MOVE && m->use_fast_prologue_epilogue;

  /* Skip return address and error code in exception handler.  */
  offset = INCOMING_FRAME_SP_OFFSET;

  /* Skip pushed static chain.  */
  if (ix86_static_chain_on_stack)
    offset += UNITS_PER_WORD;

  /* Skip saved base pointer.  */
  if (frame_pointer_needed)
    offset += UNITS_PER_WORD;
  frame->hfp_save_offset = offset;

  /* The traditional frame pointer location is at the top of the frame.  */
  frame->hard_frame_pointer_offset = offset;

  /* Register save area */
  offset += frame->nregs * UNITS_PER_WORD;
  frame->reg_save_offset = offset;

  /* Calculate the size of the va-arg area (not including padding, if any).  */
  frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size;

  /* Also adjust stack_realign_offset for the largest alignment of
     stack slot actually used.  */
  if (stack_realign_fp
      || (cfun->machine->max_used_stack_alignment != 0
	  && (offset % cfun->machine->max_used_stack_alignment) != 0))
    {
      /* We may need a 16-byte aligned stack for the remainder of the
	 register save area, but the stack frame for the local function
	 may require a greater alignment if using AVX/2/512.  In order
	 to avoid wasting space, we first calculate the space needed for
	 the rest of the register saves, add that to the stack pointer,
	 and then realign the stack to the boundary of the start of the
	 frame for the local function.  */
      HOST_WIDE_INT space_needed = 0;
      HOST_WIDE_INT sse_reg_space_needed = 0;

      if (TARGET_64BIT)
	{
	  if (m->call_ms2sysv)
	    {
	      m->call_ms2sysv_pad_in = 0;
	      space_needed = xlogue_layout::get_instance ().get_stack_space_used ();
	    }

	  else if (frame->nsseregs)
	    /* The only ABI that has saved SSE registers (Win64) also has a
	       16-byte aligned default stack.  However, many programs violate
	       the ABI, and Wine64 forces stack realignment to compensate.  */
	    space_needed = frame->nsseregs * 16;

	  sse_reg_space_needed = space_needed = ROUND_UP (space_needed, 16);

	  /* 64-bit frame->va_arg_size should always be a multiple of 16, but
	     rounding to be pedantic.  */
	  space_needed = ROUND_UP (space_needed + frame->va_arg_size, 16);
	}
      else
	space_needed = frame->va_arg_size;

      /* Record the allocation size required prior to the realignment AND.  */
      frame->stack_realign_allocate = space_needed;

      /* The re-aligned stack starts at frame->stack_realign_offset.  Values
	 before this point are not directly comparable with values below
	 this point.  Use sp_valid_at to determine if the stack pointer is
	 valid for a given offset, fp_valid_at for the frame pointer, or
	 choose_baseaddr to have a base register chosen for you.

	 Note that the result of (frame->stack_realign_offset
	 & (stack_alignment_needed - 1)) may not equal zero.  */
      offset = ROUND_UP (offset + space_needed, stack_alignment_needed);
      frame->stack_realign_offset = offset - space_needed;
      frame->sse_reg_save_offset = frame->stack_realign_offset
							+ sse_reg_space_needed;
    }
  else
    {
      frame->stack_realign_offset = offset;

      if (TARGET_64BIT && m->call_ms2sysv)
	{
	  m->call_ms2sysv_pad_in = !!(offset & UNITS_PER_WORD);
	  offset += xlogue_layout::get_instance ().get_stack_space_used ();
	}

      /* Align and set SSE register save area.  */
      else if (frame->nsseregs)
	{
	  /* If the incoming stack boundary is at least 16 bytes, or DRAP is
	     required and the DRAP re-alignment boundary is at least 16 bytes,
	     then we want the SSE register save area properly aligned.  */
	  if (ix86_incoming_stack_boundary >= 128
		  || (stack_realign_drap && stack_alignment_needed >= 16))
	    offset = ROUND_UP (offset, 16);
	  offset += frame->nsseregs * 16;
	}
      frame->sse_reg_save_offset = offset;
      offset += frame->va_arg_size;
    }

  /* Align start of frame for local function.  When a function call
     is removed, it may become a leaf function.  But if argument may
     be passed on stack, we need to align the stack when there is no
     tail call.  */
  if (m->call_ms2sysv
      || frame->va_arg_size != 0
      || size != 0
      || !crtl->is_leaf
      || (!crtl->tail_call_emit
	  && cfun->machine->outgoing_args_on_stack)
      || cfun->calls_alloca
      || ix86_current_function_calls_tls_descriptor)
    offset = ROUND_UP (offset, stack_alignment_needed);

  /* Frame pointer points here.  */
  frame->frame_pointer_offset = offset;

  offset += size;

  /* Add outgoing arguments area.  Can be skipped if we eliminated
     all the function calls as dead code.
     Skipping is however impossible when function calls alloca.  Alloca
     expander assumes that last crtl->outgoing_args_size
     of stack frame are unused.  */
  if (ACCUMULATE_OUTGOING_ARGS
      && (!crtl->is_leaf || cfun->calls_alloca
	  || ix86_current_function_calls_tls_descriptor))
    {
      offset += crtl->outgoing_args_size;
      frame->outgoing_arguments_size = crtl->outgoing_args_size;
    }
  else
    frame->outgoing_arguments_size = 0;

  /* Align stack boundary.  Only needed if we're calling another function
     or using alloca.  */
  if (!crtl->is_leaf || cfun->calls_alloca
      || ix86_current_function_calls_tls_descriptor)
    offset = ROUND_UP (offset, preferred_alignment);

  /* We've reached end of stack frame.  */
  frame->stack_pointer_offset = offset;

  /* Size prologue needs to allocate.  */
  to_allocate = offset - frame->sse_reg_save_offset;

  if ((!to_allocate && frame->nregs <= 1)
      || (TARGET_64BIT && to_allocate >= HOST_WIDE_INT_C (0x80000000))
       /* If static stack checking is enabled and done with probes,
	  the registers need to be saved before allocating the frame.  */
      || flag_stack_check == STATIC_BUILTIN_STACK_CHECK
      /* If stack clash probing needs a loop, then it needs a
	 scratch register.  But the returned register is only guaranteed
	 to be safe to use after register saves are complete.  So if
	 stack clash protections are enabled and the allocated frame is
	 larger than the probe interval, then use pushes to save
	 callee saved registers.  */
      || (flag_stack_clash_protection
	  && !ix86_target_stack_probe ()
	  && to_allocate > get_probe_interval ()))
    frame->save_regs_using_mov = false;

  if (ix86_using_red_zone ()
      && crtl->sp_is_unchanging
      && crtl->is_leaf
      && !ix86_pc_thunk_call_expanded
      && !ix86_current_function_calls_tls_descriptor)
    {
      frame->red_zone_size = to_allocate;
      if (frame->save_regs_using_mov)
	frame->red_zone_size += frame->nregs * UNITS_PER_WORD;
      if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE)
	frame->red_zone_size = RED_ZONE_SIZE - RED_ZONE_RESERVE;
    }
  else
    frame->red_zone_size = 0;
  frame->stack_pointer_offset -= frame->red_zone_size;

  /* The SEH frame pointer location is near the bottom of the frame.
     This is enforced by the fact that the difference between the
     stack pointer and the frame pointer is limited to 240 bytes in
     the unwind data structure.  */
  if (TARGET_SEH)
    {
      /* Force the frame pointer to point at or below the lowest register save
	 area, see the SEH code in config/i386/winnt.cc for the rationale.  */
      frame->hard_frame_pointer_offset = frame->sse_reg_save_offset;

      /* If we can leave the frame pointer where it is, do so; however return
	 the establisher frame for __builtin_frame_address (0) or else if the
	 frame overflows the SEH maximum frame size.

	 Note that the value returned by __builtin_frame_address (0) is quite
	 constrained, because setjmp is piggybacked on the SEH machinery with
	 recent versions of MinGW:

	  #    elif defined(__SEH__)
	  #     if defined(__aarch64__) || defined(_ARM64_)
	  #      define setjmp(BUF) _setjmp((BUF), __builtin_sponentry())
	  #     elif (__MINGW_GCC_VERSION < 40702)
	  #      define setjmp(BUF) _setjmp((BUF), mingw_getsp())
	  #     else
	  #      define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0))
	  #     endif

	 and the second argument passed to _setjmp, if not null, is forwarded
	 to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has
	 built an ExceptionRecord on the fly describing the setjmp buffer).  */
      const HOST_WIDE_INT diff
	= frame->stack_pointer_offset - frame->hard_frame_pointer_offset;
      if (diff <= 255 && !crtl->accesses_prior_frames)
	{
	  /* The resulting diff will be a multiple of 16 lower than 255,
	     i.e. at most 240 as required by the unwind data structure.  */
	  frame->hard_frame_pointer_offset += (diff & 15);
	}
      else if (diff <= SEH_MAX_FRAME_SIZE && !crtl->accesses_prior_frames)
	{
	  /* Ideally we'd determine what portion of the local stack frame
	     (within the constraint of the lowest 240) is most heavily used.
	     But without that complication, simply bias the frame pointer
	     by 128 bytes so as to maximize the amount of the local stack
	     frame that is addressable with 8-bit offsets.  */
	  frame->hard_frame_pointer_offset = frame->stack_pointer_offset - 128;
	}
      else
	frame->hard_frame_pointer_offset = frame->hfp_save_offset;
    }
}

/* This is semi-inlined memory_address_length, but simplified
   since we know that we're always dealing with reg+offset, and
   to avoid having to create and discard all that rtl.  */

static inline int
choose_baseaddr_len (unsigned int regno, HOST_WIDE_INT offset)
{
  int len = 4;

  if (offset == 0)
    {
      /* EBP and R13 cannot be encoded without an offset.  */
      len = (regno == BP_REG || regno == R13_REG);
    }
  else if (IN_RANGE (offset, -128, 127))
    len = 1;

  /* ESP and R12 must be encoded with a SIB byte.  */
  if (regno == SP_REG || regno == R12_REG)
    len++;

  return len;
}

/* Determine if the stack pointer is valid for accessing the CFA_OFFSET in
   the frame save area.  The register is saved at CFA - CFA_OFFSET.  */

static bool
sp_valid_at (HOST_WIDE_INT cfa_offset)
{
  const struct machine_frame_state &fs = cfun->machine->fs;
  if (fs.sp_realigned && cfa_offset <= fs.sp_realigned_offset)
    {
      /* Validate that the cfa_offset isn't in a "no-man's land".  */
      gcc_assert (cfa_offset <= fs.sp_realigned_fp_last);
      return false;
    }
  return fs.sp_valid;
}

/* Determine if the frame pointer is valid for accessing the CFA_OFFSET in
   the frame save area.  The register is saved at CFA - CFA_OFFSET.  */

static inline bool
fp_valid_at (HOST_WIDE_INT cfa_offset)
{
  const struct machine_frame_state &fs = cfun->machine->fs;
  if (fs.sp_realigned && cfa_offset > fs.sp_realigned_fp_last)
    {
      /* Validate that the cfa_offset isn't in a "no-man's land".  */
      gcc_assert (cfa_offset >= fs.sp_realigned_offset);
      return false;
    }
  return fs.fp_valid;
}

/* Choose a base register based upon alignment requested, speed and/or
   size.  */

static void
choose_basereg (HOST_WIDE_INT cfa_offset, rtx &base_reg,
		HOST_WIDE_INT &base_offset,
		unsigned int align_reqested, unsigned int *align)
{
  const struct machine_function *m = cfun->machine;
  unsigned int hfp_align;
  unsigned int drap_align;
  unsigned int sp_align;
  bool hfp_ok  = fp_valid_at (cfa_offset);
  bool drap_ok = m->fs.drap_valid;
  bool sp_ok   = sp_valid_at (cfa_offset);

  hfp_align = drap_align = sp_align = INCOMING_STACK_BOUNDARY;

  /* Filter out any registers that don't meet the requested alignment
     criteria.  */
  if (align_reqested)
    {
      if (m->fs.realigned)
	hfp_align = drap_align = sp_align = crtl->stack_alignment_needed;
      /* SEH unwind code does do not currently support REG_CFA_EXPRESSION
	 notes (which we would need to use a realigned stack pointer),
	 so disable on SEH targets.  */
      else if (m->fs.sp_realigned)
	sp_align = crtl->stack_alignment_needed;

      hfp_ok = hfp_ok && hfp_align >= align_reqested;
      drap_ok = drap_ok && drap_align >= align_reqested;
      sp_ok = sp_ok && sp_align >= align_reqested;
    }

  if (m->use_fast_prologue_epilogue)
    {
      /* Choose the base register most likely to allow the most scheduling
         opportunities.  Generally FP is valid throughout the function,
         while DRAP must be reloaded within the epilogue.  But choose either
         over the SP due to increased encoding size.  */

      if (hfp_ok)
	{
	  base_reg = hard_frame_pointer_rtx;
	  base_offset = m->fs.fp_offset - cfa_offset;
	}
      else if (drap_ok)
	{
	  base_reg = crtl->drap_reg;
	  base_offset = 0 - cfa_offset;
	}
      else if (sp_ok)
	{
	  base_reg = stack_pointer_rtx;
	  base_offset = m->fs.sp_offset - cfa_offset;
	}
    }
  else
    {
      HOST_WIDE_INT toffset;
      int len = 16, tlen;

      /* Choose the base register with the smallest address encoding.
         With a tie, choose FP > DRAP > SP.  */
      if (sp_ok)
	{
	  base_reg = stack_pointer_rtx;
	  base_offset = m->fs.sp_offset - cfa_offset;
          len = choose_baseaddr_len (STACK_POINTER_REGNUM, base_offset);
	}
      if (drap_ok)
	{
	  toffset = 0 - cfa_offset;
	  tlen = choose_baseaddr_len (REGNO (crtl->drap_reg), toffset);
	  if (tlen <= len)
	    {
	      base_reg = crtl->drap_reg;
	      base_offset = toffset;
	      len = tlen;
	    }
	}
      if (hfp_ok)
	{
	  toffset = m->fs.fp_offset - cfa_offset;
	  tlen = choose_baseaddr_len (HARD_FRAME_POINTER_REGNUM, toffset);
	  if (tlen <= len)
	    {
	      base_reg = hard_frame_pointer_rtx;
	      base_offset = toffset;
	    }
	}
    }

    /* Set the align return value.  */
    if (align)
      {
	if (base_reg == stack_pointer_rtx)
	  *align = sp_align;
	else if (base_reg == crtl->drap_reg)
	  *align = drap_align;
	else if (base_reg == hard_frame_pointer_rtx)
	  *align = hfp_align;
      }
}

/* Return an RTX that points to CFA_OFFSET within the stack frame and
   the alignment of address.  If ALIGN is non-null, it should point to
   an alignment value (in bits) that is preferred or zero and will
   recieve the alignment of the base register that was selected,
   irrespective of rather or not CFA_OFFSET is a multiple of that
   alignment value.  If it is possible for the base register offset to be
   non-immediate then SCRATCH_REGNO should specify a scratch register to
   use.

   The valid base registers are taken from CFUN->MACHINE->FS.  */

static rtx
choose_baseaddr (HOST_WIDE_INT cfa_offset, unsigned int *align,
		 unsigned int scratch_regno = INVALID_REGNUM)
{
  rtx base_reg = NULL;
  HOST_WIDE_INT base_offset = 0;

  /* If a specific alignment is requested, try to get a base register
     with that alignment first.  */
  if (align && *align)
    choose_basereg (cfa_offset, base_reg, base_offset, *align, align);

  if (!base_reg)
    choose_basereg (cfa_offset, base_reg, base_offset, 0, align);

  gcc_assert (base_reg != NULL);

  rtx base_offset_rtx = GEN_INT (base_offset);

  if (!x86_64_immediate_operand (base_offset_rtx, Pmode))
    {
      gcc_assert (scratch_regno != INVALID_REGNUM);

      rtx scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
      emit_move_insn (scratch_reg, base_offset_rtx);

      return gen_rtx_PLUS (Pmode, base_reg, scratch_reg);
    }

  return plus_constant (Pmode, base_reg, base_offset);
}

/* Emit code to save registers in the prologue.  */

static void
ix86_emit_save_regs (void)
{
  int regno;
  rtx_insn *insn;

  for (regno = FIRST_PSEUDO_REGISTER - 1; regno >= 0; regno--)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
      {
	insn = emit_insn (gen_push (gen_rtx_REG (word_mode, regno)));
	RTX_FRAME_RELATED_P (insn) = 1;
      }
}

/* Emit a single register save at CFA - CFA_OFFSET.  */

static void
ix86_emit_save_reg_using_mov (machine_mode mode, unsigned int regno,
			      HOST_WIDE_INT cfa_offset)
{
  struct machine_function *m = cfun->machine;
  rtx reg = gen_rtx_REG (mode, regno);
  rtx mem, addr, base, insn;
  unsigned int align = GET_MODE_ALIGNMENT (mode);

  addr = choose_baseaddr (cfa_offset, &align);
  mem = gen_frame_mem (mode, addr);

  /* The location aligment depends upon the base register.  */
  align = MIN (GET_MODE_ALIGNMENT (mode), align);
  gcc_assert (! (cfa_offset & (align / BITS_PER_UNIT - 1)));
  set_mem_align (mem, align);

  insn = emit_insn (gen_rtx_SET (mem, reg));
  RTX_FRAME_RELATED_P (insn) = 1;

  base = addr;
  if (GET_CODE (base) == PLUS)
    base = XEXP (base, 0);
  gcc_checking_assert (REG_P (base));

  /* When saving registers into a re-aligned local stack frame, avoid
     any tricky guessing by dwarf2out.  */
  if (m->fs.realigned)
    {
      gcc_checking_assert (stack_realign_drap);

      if (regno == REGNO (crtl->drap_reg))
	{
	  /* A bit of a hack.  We force the DRAP register to be saved in
	     the re-aligned stack frame, which provides us with a copy
	     of the CFA that will last past the prologue.  Install it.  */
	  gcc_checking_assert (cfun->machine->fs.fp_valid);
	  addr = plus_constant (Pmode, hard_frame_pointer_rtx,
				cfun->machine->fs.fp_offset - cfa_offset);
	  mem = gen_rtx_MEM (mode, addr);
	  add_reg_note (insn, REG_CFA_DEF_CFA, mem);
	}
      else
	{
	  /* The frame pointer is a stable reference within the
	     aligned frame.  Use it.  */
	  gcc_checking_assert (cfun->machine->fs.fp_valid);
	  addr = plus_constant (Pmode, hard_frame_pointer_rtx,
				cfun->machine->fs.fp_offset - cfa_offset);
	  mem = gen_rtx_MEM (mode, addr);
	  add_reg_note (insn, REG_CFA_EXPRESSION, gen_rtx_SET (mem, reg));
	}
    }

  else if (base == stack_pointer_rtx && m->fs.sp_realigned
	   && cfa_offset >= m->fs.sp_realigned_offset)
    {
      gcc_checking_assert (stack_realign_fp);
      add_reg_note (insn, REG_CFA_EXPRESSION, gen_rtx_SET (mem, reg));
    }

  /* The memory may not be relative to the current CFA register,
     which means that we may need to generate a new pattern for
     use by the unwind info.  */
  else if (base != m->fs.cfa_reg)
    {
      addr = plus_constant (Pmode, m->fs.cfa_reg,
			    m->fs.cfa_offset - cfa_offset);
      mem = gen_rtx_MEM (mode, addr);
      add_reg_note (insn, REG_CFA_OFFSET, gen_rtx_SET (mem, reg));
    }
}

/* Emit code to save registers using MOV insns.
   First register is stored at CFA - CFA_OFFSET.  */
static void
ix86_emit_save_regs_using_mov (HOST_WIDE_INT cfa_offset)
{
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
      {
        ix86_emit_save_reg_using_mov (word_mode, regno, cfa_offset);
	cfa_offset -= UNITS_PER_WORD;
      }
}

/* Emit code to save SSE registers using MOV insns.
   First register is stored at CFA - CFA_OFFSET.  */
static void
ix86_emit_save_sse_regs_using_mov (HOST_WIDE_INT cfa_offset)
{
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true, true))
      {
	ix86_emit_save_reg_using_mov (V4SFmode, regno, cfa_offset);
	cfa_offset -= GET_MODE_SIZE (V4SFmode);
      }
}

static GTY(()) rtx queued_cfa_restores;

/* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
   manipulation insn.  The value is on the stack at CFA - CFA_OFFSET.
   Don't add the note if the previously saved value will be left untouched
   within stack red-zone till return, as unwinders can find the same value
   in the register and on the stack.  */

static void
ix86_add_cfa_restore_note (rtx_insn *insn, rtx reg, HOST_WIDE_INT cfa_offset)
{
  if (!crtl->shrink_wrapped
      && cfa_offset <= cfun->machine->fs.red_zone_offset)
    return;

  if (insn)
    {
      add_reg_note (insn, REG_CFA_RESTORE, reg);
      RTX_FRAME_RELATED_P (insn) = 1;
    }
  else
    queued_cfa_restores
      = alloc_reg_note (REG_CFA_RESTORE, reg, queued_cfa_restores);
}

/* Add queued REG_CFA_RESTORE notes if any to INSN.  */

static void
ix86_add_queued_cfa_restore_notes (rtx insn)
{
  rtx last;
  if (!queued_cfa_restores)
    return;
  for (last = queued_cfa_restores; XEXP (last, 1); last = XEXP (last, 1))
    ;
  XEXP (last, 1) = REG_NOTES (insn);
  REG_NOTES (insn) = queued_cfa_restores;
  queued_cfa_restores = NULL_RTX;
  RTX_FRAME_RELATED_P (insn) = 1;
}

/* Expand prologue or epilogue stack adjustment.
   The pattern exist to put a dependency on all ebp-based memory accesses.
   STYLE should be negative if instructions should be marked as frame related,
   zero if %r11 register is live and cannot be freely used and positive
   otherwise.  */

static rtx
pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset,
			   int style, bool set_cfa)
{
  struct machine_function *m = cfun->machine;
  rtx addend = offset;
  rtx insn;
  bool add_frame_related_expr = false;

  if (!x86_64_immediate_operand (offset, Pmode))
    {
      /* r11 is used by indirect sibcall return as well, set before the
	 epilogue and used after the epilogue.  */
      if (style)
        addend = gen_rtx_REG (Pmode, R11_REG);
      else
	{
	  gcc_assert (src != hard_frame_pointer_rtx
		      && dest != hard_frame_pointer_rtx);
	  addend = hard_frame_pointer_rtx;
	}
      emit_insn (gen_rtx_SET (addend, offset));
      if (style < 0)
	add_frame_related_expr = true;
    }

  insn = emit_insn (gen_pro_epilogue_adjust_stack_add
		    (Pmode, dest, src, addend));
  if (style >= 0)
    ix86_add_queued_cfa_restore_notes (insn);

  if (set_cfa)
    {
      rtx r;

      gcc_assert (m->fs.cfa_reg == src);
      m->fs.cfa_offset += INTVAL (offset);
      m->fs.cfa_reg = dest;

      r = gen_rtx_PLUS (Pmode, src, offset);
      r = gen_rtx_SET (dest, r);
      add_reg_note (insn, REG_CFA_ADJUST_CFA, r);
      RTX_FRAME_RELATED_P (insn) = 1;
    }
  else if (style < 0)
    {
      RTX_FRAME_RELATED_P (insn) = 1;
      if (add_frame_related_expr)
	{
	  rtx r = gen_rtx_PLUS (Pmode, src, offset);
	  r = gen_rtx_SET (dest, r);
	  add_reg_note (insn, REG_FRAME_RELATED_EXPR, r);
	}
    }

  if (dest == stack_pointer_rtx)
    {
      HOST_WIDE_INT ooffset = m->fs.sp_offset;
      bool valid = m->fs.sp_valid;
      bool realigned = m->fs.sp_realigned;

      if (src == hard_frame_pointer_rtx)
	{
	  valid = m->fs.fp_valid;
	  realigned = false;
	  ooffset = m->fs.fp_offset;
	}
      else if (src == crtl->drap_reg)
	{
	  valid = m->fs.drap_valid;
	  realigned = false;
	  ooffset = 0;
	}
      else
	{
	  /* Else there are two possibilities: SP itself, which we set
	     up as the default above.  Or EH_RETURN_STACKADJ_RTX, which is
	     taken care of this by hand along the eh_return path.  */
	  gcc_checking_assert (src == stack_pointer_rtx
			       || offset == const0_rtx);
	}

      m->fs.sp_offset = ooffset - INTVAL (offset);
      m->fs.sp_valid = valid;
      m->fs.sp_realigned = realigned;
    }
  return insn;
}

/* Find an available register to be used as dynamic realign argument
   pointer regsiter.  Such a register will be written in prologue and
   used in begin of body, so it must not be
	1. parameter passing register.
	2. GOT pointer.
   We reuse static-chain register if it is available.  Otherwise, we
   use DI for i386 and R13 for x86-64.  We chose R13 since it has
   shorter encoding.

   Return: the regno of chosen register.  */

static unsigned int
find_drap_reg (void)
{
  tree decl = cfun->decl;

  /* Always use callee-saved register if there are no caller-saved
     registers.  */
  if (TARGET_64BIT)
    {
      /* Use R13 for nested function or function need static chain.
	 Since function with tail call may use any caller-saved
	 registers in epilogue, DRAP must not use caller-saved
	 register in such case.  */
      if (DECL_STATIC_CHAIN (decl)
	  || cfun->machine->no_caller_saved_registers
	  || crtl->tail_call_emit)
	return R13_REG;

      return R10_REG;
    }
  else
    {
      /* Use DI for nested function or function need static chain.
	 Since function with tail call may use any caller-saved
	 registers in epilogue, DRAP must not use caller-saved
	 register in such case.  */
      if (DECL_STATIC_CHAIN (decl)
	  || cfun->machine->no_caller_saved_registers
	  || crtl->tail_call_emit
	  || crtl->calls_eh_return)
	return DI_REG;

      /* Reuse static chain register if it isn't used for parameter
         passing.  */
      if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2)
	{
	  unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (decl));
	  if ((ccvt & (IX86_CALLCVT_FASTCALL | IX86_CALLCVT_THISCALL)) == 0)
	    return CX_REG;
	}
      return DI_REG;
    }
}

/* Return minimum incoming stack alignment.  */

static unsigned int
ix86_minimum_incoming_stack_boundary (bool sibcall)
{
  unsigned int incoming_stack_boundary;

  /* Stack of interrupt handler is aligned to 128 bits in 64bit mode.  */
  if (cfun->machine->func_type != TYPE_NORMAL)
    incoming_stack_boundary = TARGET_64BIT ? 128 : MIN_STACK_BOUNDARY;
  /* Prefer the one specified at command line. */
  else if (ix86_user_incoming_stack_boundary)
    incoming_stack_boundary = ix86_user_incoming_stack_boundary;
  /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
     if -mstackrealign is used, it isn't used for sibcall check and
     estimated stack alignment is 128bit.  */
  else if (!sibcall
	   && ix86_force_align_arg_pointer
	   && crtl->stack_alignment_estimated == 128)
    incoming_stack_boundary = MIN_STACK_BOUNDARY;
  else
    incoming_stack_boundary = ix86_default_incoming_stack_boundary;

  /* Incoming stack alignment can be changed on individual functions
     via force_align_arg_pointer attribute.  We use the smallest
     incoming stack boundary.  */
  if (incoming_stack_boundary > MIN_STACK_BOUNDARY
      && lookup_attribute ("force_align_arg_pointer",
			   TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
    incoming_stack_boundary = MIN_STACK_BOUNDARY;

  /* The incoming stack frame has to be aligned at least at
     parm_stack_boundary.  */
  if (incoming_stack_boundary < crtl->parm_stack_boundary)
    incoming_stack_boundary = crtl->parm_stack_boundary;

  /* Stack at entrance of main is aligned by runtime.  We use the
     smallest incoming stack boundary. */
  if (incoming_stack_boundary > MAIN_STACK_BOUNDARY
      && DECL_NAME (current_function_decl)
      && MAIN_NAME_P (DECL_NAME (current_function_decl))
      && DECL_FILE_SCOPE_P (current_function_decl))
    incoming_stack_boundary = MAIN_STACK_BOUNDARY;

  return incoming_stack_boundary;
}

/* Update incoming stack boundary and estimated stack alignment.  */

static void
ix86_update_stack_boundary (void)
{
  ix86_incoming_stack_boundary
    = ix86_minimum_incoming_stack_boundary (false);

  /* x86_64 vararg needs 16byte stack alignment for register save area.  */
  if (TARGET_64BIT
      && cfun->stdarg
      && crtl->stack_alignment_estimated < 128)
    crtl->stack_alignment_estimated = 128;

  /* __tls_get_addr needs to be called with 16-byte aligned stack.  */
  if (ix86_tls_descriptor_calls_expanded_in_cfun
      && crtl->preferred_stack_boundary < 128)
    crtl->preferred_stack_boundary = 128;
}

/* Handle the TARGET_GET_DRAP_RTX hook.  Return NULL if no DRAP is
   needed or an rtx for DRAP otherwise.  */

static rtx
ix86_get_drap_rtx (void)
{
  /* We must use DRAP if there are outgoing arguments on stack or
     the stack pointer register is clobbered by asm statment and
     ACCUMULATE_OUTGOING_ARGS is false.  */
  if (ix86_force_drap
      || ((cfun->machine->outgoing_args_on_stack
	   || crtl->sp_is_clobbered_by_asm)
	  && !ACCUMULATE_OUTGOING_ARGS))
    crtl->need_drap = true;

  if (stack_realign_drap)
    {
      /* Assign DRAP to vDRAP and returns vDRAP */
      unsigned int regno = find_drap_reg ();
      rtx drap_vreg;
      rtx arg_ptr;
      rtx_insn *seq, *insn;

      arg_ptr = gen_rtx_REG (Pmode, regno);
      crtl->drap_reg = arg_ptr;

      start_sequence ();
      drap_vreg = copy_to_reg (arg_ptr);
      seq = get_insns ();
      end_sequence ();

      insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
      if (!optimize)
	{
	  add_reg_note (insn, REG_CFA_SET_VDRAP, drap_vreg);
	  RTX_FRAME_RELATED_P (insn) = 1;
	}
      return drap_vreg;
    }
  else
    return NULL;
}

/* Handle the TARGET_INTERNAL_ARG_POINTER hook.  */

static rtx
ix86_internal_arg_pointer (void)
{
  return virtual_incoming_args_rtx;
}

struct scratch_reg {
  rtx reg;
  bool saved;
};

/* Return a short-lived scratch register for use on function entry.
   In 32-bit mode, it is valid only after the registers are saved
   in the prologue.  This register must be released by means of
   release_scratch_register_on_entry once it is dead.  */

static void
get_scratch_register_on_entry (struct scratch_reg *sr)
{
  int regno;

  sr->saved = false;

  if (TARGET_64BIT)
    {
      /* We always use R11 in 64-bit mode.  */
      regno = R11_REG;
    }
  else
    {
      tree decl = current_function_decl, fntype = TREE_TYPE (decl);
      bool fastcall_p
	= lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
      bool thiscall_p
	= lookup_attribute ("thiscall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
      bool static_chain_p = DECL_STATIC_CHAIN (decl);
      int regparm = ix86_function_regparm (fntype, decl);
      int drap_regno
	= crtl->drap_reg ? REGNO (crtl->drap_reg) : INVALID_REGNUM;

      /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
	  for the static chain register.  */
      if ((regparm < 1 || (fastcall_p && !static_chain_p))
	  && drap_regno != AX_REG)
	regno = AX_REG;
      /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
	  for the static chain register.  */
      else if (thiscall_p && !static_chain_p && drap_regno != AX_REG)
        regno = AX_REG;
      else if (regparm < 2 && !thiscall_p && drap_regno != DX_REG)
	regno = DX_REG;
      /* ecx is the static chain register.  */
      else if (regparm < 3 && !fastcall_p && !thiscall_p
	       && !static_chain_p
	       && drap_regno != CX_REG)
	regno = CX_REG;
      else if (ix86_save_reg (BX_REG, true, false))
	regno = BX_REG;
      /* esi is the static chain register.  */
      else if (!(regparm == 3 && static_chain_p)
	       && ix86_save_reg (SI_REG, true, false))
	regno = SI_REG;
      else if (ix86_save_reg (DI_REG, true, false))
	regno = DI_REG;
      else
	{
	  regno = (drap_regno == AX_REG ? DX_REG : AX_REG);
	  sr->saved = true;
	}
    }

  sr->reg = gen_rtx_REG (Pmode, regno);
  if (sr->saved)
    {
      rtx_insn *insn = emit_insn (gen_push (sr->reg));
      RTX_FRAME_RELATED_P (insn) = 1;
    }
}

/* Release a scratch register obtained from the preceding function.

   If RELEASE_VIA_POP is true, we just pop the register off the stack
   to release it.  This is what non-Linux systems use with -fstack-check.

   Otherwise we use OFFSET to locate the saved register and the
   allocated stack space becomes part of the local frame and is
   deallocated by the epilogue.  */

static void
release_scratch_register_on_entry (struct scratch_reg *sr, HOST_WIDE_INT offset,
				   bool release_via_pop)
{
  if (sr->saved)
    {
      if (release_via_pop)
	{
	  struct machine_function *m = cfun->machine;
	  rtx x, insn = emit_insn (gen_pop (sr->reg));

	  /* The RX FRAME_RELATED_P mechanism doesn't know about pop.  */
	  RTX_FRAME_RELATED_P (insn) = 1;
	  x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
	  x = gen_rtx_SET (stack_pointer_rtx, x);
	  add_reg_note (insn, REG_FRAME_RELATED_EXPR, x);
	  m->fs.sp_offset -= UNITS_PER_WORD;
	}
      else
	{
	  rtx x = plus_constant (Pmode, stack_pointer_rtx, offset);
	  x = gen_rtx_SET (sr->reg, gen_rtx_MEM (word_mode, x));
	  emit_insn (x);
	}
    }
}

/* Emit code to adjust the stack pointer by SIZE bytes while probing it.

   If INT_REGISTERS_SAVED is true, then integer registers have already been
   pushed on the stack.

   If PROTECTION AREA is true, then probe PROBE_INTERVAL plus a small dope
   beyond SIZE bytes.

   This assumes no knowledge of the current probing state, i.e. it is never
   allowed to allocate more than PROBE_INTERVAL bytes of stack space without
   a suitable probe.  */

static void
ix86_adjust_stack_and_probe (HOST_WIDE_INT size,
			     const bool int_registers_saved,
			     const bool protection_area)
{
  struct machine_function *m = cfun->machine;

  /* If this function does not statically allocate stack space, then
     no probes are needed.  */
  if (!size)
    {
      /* However, the allocation of space via pushes for register
	 saves could be viewed as allocating space, but without the
	 need to probe.  */
      if (m->frame.nregs || m->frame.nsseregs || frame_pointer_needed)
        dump_stack_clash_frame_info (NO_PROBE_SMALL_FRAME, true);
      else
	dump_stack_clash_frame_info (NO_PROBE_NO_FRAME, false);
      return;
    }

  /* If we are a noreturn function, then we have to consider the
     possibility that we're called via a jump rather than a call.

     Thus we don't have the implicit probe generated by saving the
     return address into the stack at the call.  Thus, the stack
     pointer could be anywhere in the guard page.  The safe thing
     to do is emit a probe now.

     The probe can be avoided if we have already emitted any callee
     register saves into the stack or have a frame pointer (which will
     have been saved as well).  Those saves will function as implicit
     probes.

     ?!? This should be revamped to work like aarch64 and s390 where
     we track the offset from the most recent probe.  Normally that
     offset would be zero.  For a noreturn function we would reset
     it to PROBE_INTERVAL - (STACK_BOUNDARY / BITS_PER_UNIT).   Then
     we just probe when we cross PROBE_INTERVAL.  */
  if (TREE_THIS_VOLATILE (cfun->decl)
      && !(m->frame.nregs || m->frame.nsseregs || frame_pointer_needed))
    {
      /* We can safely use any register here since we're just going to push
	 its value and immediately pop it back.  But we do try and avoid
	 argument passing registers so as not to introduce dependencies in
	 the pipeline.  For 32 bit we use %esi and for 64 bit we use %rax.  */
      rtx dummy_reg = gen_rtx_REG (word_mode, TARGET_64BIT ? AX_REG : SI_REG);
      rtx_insn *insn_push = emit_insn (gen_push (dummy_reg));
      rtx_insn *insn_pop = emit_insn (gen_pop (dummy_reg));
      m->fs.sp_offset -= UNITS_PER_WORD;
      if (m->fs.cfa_reg == stack_pointer_rtx)
	{
	  m->fs.cfa_offset -= UNITS_PER_WORD;
	  rtx x = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
	  x = gen_rtx_SET (stack_pointer_rtx, x);
	  add_reg_note (insn_push, REG_CFA_ADJUST_CFA, x);
	  RTX_FRAME_RELATED_P (insn_push) = 1;
	  x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
	  x = gen_rtx_SET (stack_pointer_rtx, x);
	  add_reg_note (insn_pop, REG_CFA_ADJUST_CFA, x);
	  RTX_FRAME_RELATED_P (insn_pop) = 1;
	}
      emit_insn (gen_blockage ());
    }

  const HOST_WIDE_INT probe_interval = get_probe_interval ();
  const int dope = 4 * UNITS_PER_WORD;

  /* If there is protection area, take it into account in the size.  */
  if (protection_area)
    size += probe_interval + dope;

  /* If we allocate less than the size of the guard statically,
     then no probing is necessary, but we do need to allocate
     the stack.  */
  else if (size < (1 << param_stack_clash_protection_guard_size))
    {
      pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
			         GEN_INT (-size), -1,
			         m->fs.cfa_reg == stack_pointer_rtx);
      dump_stack_clash_frame_info (NO_PROBE_SMALL_FRAME, true);
      return;
    }

  /* We're allocating a large enough stack frame that we need to
     emit probes.  Either emit them inline or in a loop depending
     on the size.  */
  if (size <= 4 * probe_interval)
    {
      HOST_WIDE_INT i;
      for (i = probe_interval; i <= size; i += probe_interval)
	{
	  /* Allocate PROBE_INTERVAL bytes.  */
	  rtx insn
	    = pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
					 GEN_INT (-probe_interval), -1,
					 m->fs.cfa_reg == stack_pointer_rtx);
	  add_reg_note (insn, REG_STACK_CHECK, const0_rtx);

	  /* And probe at *sp.  */
	  emit_stack_probe (stack_pointer_rtx);
	  emit_insn (gen_blockage ());
	}

      /* We need to allocate space for the residual, but we do not need
	 to probe the residual...  */
      HOST_WIDE_INT residual = (i - probe_interval - size);
      if (residual)
	{
	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				     GEN_INT (residual), -1,
				     m->fs.cfa_reg == stack_pointer_rtx);

	  /* ...except if there is a protection area to maintain.  */
	  if (protection_area)
	    emit_stack_probe (stack_pointer_rtx);
	}

      dump_stack_clash_frame_info (PROBE_INLINE, residual != 0);
    }
  else
    {
      /* We expect the GP registers to be saved when probes are used
	 as the probing sequences might need a scratch register and
	 the routine to allocate one assumes the integer registers
	 have already been saved.  */
      gcc_assert (int_registers_saved);

      struct scratch_reg sr;
      get_scratch_register_on_entry (&sr);

      /* If we needed to save a register, then account for any space
	 that was pushed (we are not going to pop the register when
	 we do the restore).  */
      if (sr.saved)
	size -= UNITS_PER_WORD;

      /* Step 1: round SIZE down to a multiple of the interval.  */
      HOST_WIDE_INT rounded_size = size & -probe_interval;

      /* Step 2: compute final value of the loop counter.  Use lea if
	 possible.  */
      rtx addr = plus_constant (Pmode, stack_pointer_rtx, -rounded_size);
      rtx insn;
      if (address_no_seg_operand (addr, Pmode))
	insn = emit_insn (gen_rtx_SET (sr.reg, addr));
      else
	{
	  emit_move_insn (sr.reg, GEN_INT (-rounded_size));
	  insn = emit_insn (gen_rtx_SET (sr.reg,
					 gen_rtx_PLUS (Pmode, sr.reg,
						       stack_pointer_rtx)));
	}
      if (m->fs.cfa_reg == stack_pointer_rtx)
	{
	  add_reg_note (insn, REG_CFA_DEF_CFA,
			plus_constant (Pmode, sr.reg,
				       m->fs.cfa_offset + rounded_size));
	  RTX_FRAME_RELATED_P (insn) = 1;
	}

      /* Step 3: the loop.  */
      rtx size_rtx = GEN_INT (rounded_size);
      insn = emit_insn (gen_adjust_stack_and_probe (Pmode, sr.reg, sr.reg,
						    size_rtx));
      if (m->fs.cfa_reg == stack_pointer_rtx)
	{
	  m->fs.cfa_offset += rounded_size;
	  add_reg_note (insn, REG_CFA_DEF_CFA,
			plus_constant (Pmode, stack_pointer_rtx,
				       m->fs.cfa_offset));
	  RTX_FRAME_RELATED_P (insn) = 1;
	}
      m->fs.sp_offset += rounded_size;
      emit_insn (gen_blockage ());

      /* Step 4: adjust SP if we cannot assert at compile-time that SIZE
	 is equal to ROUNDED_SIZE.  */

      if (size != rounded_size)
	{
	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				     GEN_INT (rounded_size - size), -1,
				     m->fs.cfa_reg == stack_pointer_rtx);

	  if (protection_area)
	    emit_stack_probe (stack_pointer_rtx);
	}

      dump_stack_clash_frame_info (PROBE_LOOP, size != rounded_size);

      /* This does not deallocate the space reserved for the scratch
	 register.  That will be deallocated in the epilogue.  */
      release_scratch_register_on_entry (&sr, size, false);
    }

  /* Adjust back to account for the protection area.  */
  if (protection_area)
    pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
			       GEN_INT (probe_interval + dope), -1,
			       m->fs.cfa_reg == stack_pointer_rtx);

  /* Make sure nothing is scheduled before we are done.  */
  emit_insn (gen_blockage ());
}

/* Adjust the stack pointer up to REG while probing it.  */

const char *
output_adjust_stack_and_probe (rtx reg)
{
  static int labelno = 0;
  char loop_lab[32];
  rtx xops[2];

  ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);

  /* Loop.  */
  ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);

  /* SP = SP + PROBE_INTERVAL.  */
  xops[0] = stack_pointer_rtx;
  xops[1] = GEN_INT (get_probe_interval ());
  output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);

  /* Probe at SP.  */
  xops[1] = const0_rtx;
  output_asm_insn ("or%z0\t{%1, (%0)|DWORD PTR [%0], %1}", xops);

  /* Test if SP == LAST_ADDR.  */
  xops[0] = stack_pointer_rtx;
  xops[1] = reg;
  output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);

  /* Branch.  */
  fputs ("\tjne\t", asm_out_file);
  assemble_name_raw (asm_out_file, loop_lab);
  fputc ('\n', asm_out_file);

  return "";
}

/* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
   inclusive.  These are offsets from the current stack pointer.

   INT_REGISTERS_SAVED is true if integer registers have already been
   pushed on the stack.  */

static void
ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size,
			     const bool int_registers_saved)
{
  const HOST_WIDE_INT probe_interval = get_probe_interval ();

  /* See if we have a constant small number of probes to generate.  If so,
     that's the easy case.  The run-time loop is made up of 6 insns in the
     generic case while the compile-time loop is made up of n insns for n #
     of intervals.  */
  if (size <= 6 * probe_interval)
    {
      HOST_WIDE_INT i;

      /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
	 it exceeds SIZE.  If only one probe is needed, this will not
	 generate any code.  Then probe at FIRST + SIZE.  */
      for (i = probe_interval; i < size; i += probe_interval)
	emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
					 -(first + i)));

      emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
				       -(first + size)));
    }

  /* Otherwise, do the same as above, but in a loop.  Note that we must be
     extra careful with variables wrapping around because we might be at
     the very top (or the very bottom) of the address space and we have
     to be able to handle this case properly; in particular, we use an
     equality test for the loop condition.  */
  else
    {
      /* We expect the GP registers to be saved when probes are used
	 as the probing sequences might need a scratch register and
	 the routine to allocate one assumes the integer registers
	 have already been saved.  */
      gcc_assert (int_registers_saved);

      HOST_WIDE_INT rounded_size, last;
      struct scratch_reg sr;

      get_scratch_register_on_entry (&sr);


      /* Step 1: round SIZE to the previous multiple of the interval.  */

      rounded_size = ROUND_DOWN (size, probe_interval);


      /* Step 2: compute initial and final value of the loop counter.  */

      /* TEST_OFFSET = FIRST.  */
      emit_move_insn (sr.reg, GEN_INT (-first));

      /* LAST_OFFSET = FIRST + ROUNDED_SIZE.  */
      last = first + rounded_size;


      /* Step 3: the loop

	 do
	   {
	     TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
	     probe at TEST_ADDR
	   }
	 while (TEST_ADDR != LAST_ADDR)

         probes at FIRST + N * PROBE_INTERVAL for values of N from 1
         until it is equal to ROUNDED_SIZE.  */

      emit_insn
	(gen_probe_stack_range (Pmode, sr.reg, sr.reg, GEN_INT (-last)));


      /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
	 that SIZE is equal to ROUNDED_SIZE.  */

      if (size != rounded_size)
	emit_stack_probe (plus_constant (Pmode,
					 gen_rtx_PLUS (Pmode,
						       stack_pointer_rtx,
						       sr.reg),
					 rounded_size - size));

      release_scratch_register_on_entry (&sr, size, true);
    }

  /* Make sure nothing is scheduled before we are done.  */
  emit_insn (gen_blockage ());
}

/* Probe a range of stack addresses from REG to END, inclusive.  These are
   offsets from the current stack pointer.  */

const char *
output_probe_stack_range (rtx reg, rtx end)
{
  static int labelno = 0;
  char loop_lab[32];
  rtx xops[3];

  ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);

  /* Loop.  */
  ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);

  /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL.  */
  xops[0] = reg;
  xops[1] = GEN_INT (get_probe_interval ());
  output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);

  /* Probe at TEST_ADDR.  */
  xops[0] = stack_pointer_rtx;
  xops[1] = reg;
  xops[2] = const0_rtx;
  output_asm_insn ("or%z0\t{%2, (%0,%1)|DWORD PTR [%0+%1], %2}", xops);

  /* Test if TEST_ADDR == LAST_ADDR.  */
  xops[0] = reg;
  xops[1] = end;
  output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);

  /* Branch.  */
  fputs ("\tjne\t", asm_out_file);
  assemble_name_raw (asm_out_file, loop_lab);
  fputc ('\n', asm_out_file);

  return "";
}

/* Set stack_frame_required to false if stack frame isn't required.
   Update STACK_ALIGNMENT to the largest alignment, in bits, of stack
   slot used if stack frame is required and CHECK_STACK_SLOT is true.  */

static void
ix86_find_max_used_stack_alignment (unsigned int &stack_alignment,
				    bool check_stack_slot)
{
  HARD_REG_SET set_up_by_prologue, prologue_used;
  basic_block bb;

  CLEAR_HARD_REG_SET (prologue_used);
  CLEAR_HARD_REG_SET (set_up_by_prologue);
  add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM);
  add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM);
  add_to_hard_reg_set (&set_up_by_prologue, Pmode,
		       HARD_FRAME_POINTER_REGNUM);

  /* The preferred stack alignment is the minimum stack alignment.  */
  if (stack_alignment > crtl->preferred_stack_boundary)
    stack_alignment = crtl->preferred_stack_boundary;

  bool require_stack_frame = false;

  FOR_EACH_BB_FN (bb, cfun)
    {
      rtx_insn *insn;
      FOR_BB_INSNS (bb, insn)
	if (NONDEBUG_INSN_P (insn)
	    && requires_stack_frame_p (insn, prologue_used,
				       set_up_by_prologue))
	  {
	    require_stack_frame = true;

	    if (check_stack_slot)
	      {
		/* Find the maximum stack alignment.  */
		subrtx_iterator::array_type array;
		FOR_EACH_SUBRTX (iter, array, PATTERN (insn), ALL)
		  if (MEM_P (*iter)
		      && (reg_mentioned_p (stack_pointer_rtx,
					   *iter)
			  || reg_mentioned_p (frame_pointer_rtx,
					      *iter)))
		    {
		      unsigned int alignment = MEM_ALIGN (*iter);
		      if (alignment > stack_alignment)
			stack_alignment = alignment;
		    }
	      }
	  }
    }

  cfun->machine->stack_frame_required = require_stack_frame;
}

/* Finalize stack_realign_needed and frame_pointer_needed flags, which
   will guide prologue/epilogue to be generated in correct form.  */

static void
ix86_finalize_stack_frame_flags (void)
{
  /* Check if stack realign is really needed after reload, and
     stores result in cfun */
  unsigned int incoming_stack_boundary
    = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
       ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
  unsigned int stack_alignment
    = (crtl->is_leaf && !ix86_current_function_calls_tls_descriptor
       ? crtl->max_used_stack_slot_alignment
       : crtl->stack_alignment_needed);
  unsigned int stack_realign
    = (incoming_stack_boundary < stack_alignment);
  bool recompute_frame_layout_p = false;

  if (crtl->stack_realign_finalized)
    {
      /* After stack_realign_needed is finalized, we can't no longer
	 change it.  */
      gcc_assert (crtl->stack_realign_needed == stack_realign);
      return;
    }

  /* It is always safe to compute max_used_stack_alignment.  We
     compute it only if 128-bit aligned load/store may be generated
     on misaligned stack slot which will lead to segfault. */
  bool check_stack_slot
    = (stack_realign || crtl->max_used_stack_slot_alignment >= 128);
  ix86_find_max_used_stack_alignment (stack_alignment,
				      check_stack_slot);

  /* If the only reason for frame_pointer_needed is that we conservatively
     assumed stack realignment might be needed or -fno-omit-frame-pointer
     is used, but in the end nothing that needed the stack alignment had
     been spilled nor stack access, clear frame_pointer_needed and say we
     don't need stack realignment.

     When vector register is used for piecewise move and store, we don't
     increase stack_alignment_needed as there is no register spill for
     piecewise move and store.  Since stack_realign_needed is set to true
     by checking stack_alignment_estimated which is updated by pseudo
     vector register usage, we also need to check stack_realign_needed to
     eliminate frame pointer.  */
  if ((stack_realign
       || (!flag_omit_frame_pointer && optimize)
       || crtl->stack_realign_needed)
      && frame_pointer_needed
      && crtl->is_leaf
      && crtl->sp_is_unchanging
      && !ix86_current_function_calls_tls_descriptor
      && !crtl->accesses_prior_frames
      && !cfun->calls_alloca
      && !crtl->calls_eh_return
      /* See ira_setup_eliminable_regset for the rationale.  */
      && !(STACK_CHECK_MOVING_SP
	   && flag_stack_check
	   && flag_exceptions
	   && cfun->can_throw_non_call_exceptions)
      && !ix86_frame_pointer_required ()
      && ix86_get_frame_size () == 0
      && ix86_nsaved_sseregs () == 0
      && ix86_varargs_gpr_size + ix86_varargs_fpr_size == 0)
    {
      if (cfun->machine->stack_frame_required)
	{
	  /* Stack frame is required.  If stack alignment needed is less
	     than incoming stack boundary, don't realign stack.  */
	  stack_realign = incoming_stack_boundary < stack_alignment;
	  if (!stack_realign)
	    {
	      crtl->max_used_stack_slot_alignment
		= incoming_stack_boundary;
	      crtl->stack_alignment_needed
		= incoming_stack_boundary;
	      /* Also update preferred_stack_boundary for leaf
	         functions.  */
	      crtl->preferred_stack_boundary
		= incoming_stack_boundary;
	    }
	}
      else
	{
	  /* If drap has been set, but it actually isn't live at the
	     start of the function, there is no reason to set it up.  */
	  if (crtl->drap_reg)
	    {
	      basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
	      if (! REGNO_REG_SET_P (DF_LR_IN (bb),
				     REGNO (crtl->drap_reg)))
		{
		  crtl->drap_reg = NULL_RTX;
		  crtl->need_drap = false;
		}
	    }
	  else
	    cfun->machine->no_drap_save_restore = true;

	  frame_pointer_needed = false;
	  stack_realign = false;
	  crtl->max_used_stack_slot_alignment = incoming_stack_boundary;
	  crtl->stack_alignment_needed = incoming_stack_boundary;
	  crtl->stack_alignment_estimated = incoming_stack_boundary;
	  if (crtl->preferred_stack_boundary > incoming_stack_boundary)
	    crtl->preferred_stack_boundary = incoming_stack_boundary;
	  df_finish_pass (true);
	  df_scan_alloc (NULL);
	  df_scan_blocks ();
	  df_compute_regs_ever_live (true);
	  df_analyze ();

	  if (flag_var_tracking)
	    {
	      /* Since frame pointer is no longer available, replace it with
		 stack pointer - UNITS_PER_WORD in debug insns.  */
	      df_ref ref, next;
	      for (ref = DF_REG_USE_CHAIN (HARD_FRAME_POINTER_REGNUM);
		   ref; ref = next)
		{
		  next = DF_REF_NEXT_REG (ref);
		  if (!DF_REF_INSN_INFO (ref))
		    continue;

		  /* Make sure the next ref is for a different instruction,
		     so that we're not affected by the rescan.  */
		  rtx_insn *insn = DF_REF_INSN (ref);
		  while (next && DF_REF_INSN (next) == insn)
		    next = DF_REF_NEXT_REG (next);

		  if (DEBUG_INSN_P (insn))
		    {
		      bool changed = false;
		      for (; ref != next; ref = DF_REF_NEXT_REG (ref))
			{
			  rtx *loc = DF_REF_LOC (ref);
			  if (*loc == hard_frame_pointer_rtx)
			    {
			      *loc = plus_constant (Pmode,
						    stack_pointer_rtx,
						    -UNITS_PER_WORD);
			      changed = true;
			    }
			}
		      if (changed)
			df_insn_rescan (insn);
		    }
		}
	    }

	  recompute_frame_layout_p = true;
	}
    }
  else if (crtl->max_used_stack_slot_alignment >= 128
	   && cfun->machine->stack_frame_required)
    {
      /* We don't need to realign stack.  max_used_stack_alignment is
	 used to decide how stack frame should be aligned.  This is
	 independent of any psABIs nor 32-bit vs 64-bit.  */
      cfun->machine->max_used_stack_alignment
	= stack_alignment / BITS_PER_UNIT;
    }

  if (crtl->stack_realign_needed != stack_realign)
    recompute_frame_layout_p = true;
  crtl->stack_realign_needed = stack_realign;
  crtl->stack_realign_finalized = true;
  if (recompute_frame_layout_p)
    ix86_compute_frame_layout ();
}

/* Delete SET_GOT right after entry block if it is allocated to reg.  */

static void
ix86_elim_entry_set_got (rtx reg)
{
  basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
  rtx_insn *c_insn = BB_HEAD (bb);
  if (!NONDEBUG_INSN_P (c_insn))
    c_insn = next_nonnote_nondebug_insn (c_insn);
  if (c_insn && NONJUMP_INSN_P (c_insn))
    {
      rtx pat = PATTERN (c_insn);
      if (GET_CODE (pat) == PARALLEL)
	{
	  rtx vec = XVECEXP (pat, 0, 0);
	  if (GET_CODE (vec) == SET
	      && XINT (XEXP (vec, 1), 1) == UNSPEC_SET_GOT
	      && REGNO (XEXP (vec, 0)) == REGNO (reg))
	    delete_insn (c_insn);
	}
    }
}

static rtx
gen_frame_set (rtx reg, rtx frame_reg, int offset, bool store)
{
  rtx addr, mem;

  if (offset)
    addr = plus_constant (Pmode, frame_reg, offset);
  mem = gen_frame_mem (GET_MODE (reg), offset ? addr : frame_reg);
  return gen_rtx_SET (store ? mem : reg, store ? reg : mem);
}

static inline rtx
gen_frame_load (rtx reg, rtx frame_reg, int offset)
{
  return gen_frame_set (reg, frame_reg, offset, false);
}

static inline rtx
gen_frame_store (rtx reg, rtx frame_reg, int offset)
{
  return gen_frame_set (reg, frame_reg, offset, true);
}

static void
ix86_emit_outlined_ms2sysv_save (const struct ix86_frame &frame)
{
  struct machine_function *m = cfun->machine;
  const unsigned ncregs = NUM_X86_64_MS_CLOBBERED_REGS
			  + m->call_ms2sysv_extra_regs;
  rtvec v = rtvec_alloc (ncregs + 1);
  unsigned int align, i, vi = 0;
  rtx_insn *insn;
  rtx sym, addr;
  rtx rax = gen_rtx_REG (word_mode, AX_REG);
  const class xlogue_layout &xlogue = xlogue_layout::get_instance ();

  /* AL should only be live with sysv_abi.  */
  gcc_assert (!ix86_eax_live_at_start_p ());
  gcc_assert (m->fs.sp_offset >= frame.sse_reg_save_offset);

  /* Setup RAX as the stub's base pointer.  We use stack_realign_offset rather
     we've actually realigned the stack or not.  */
  align = GET_MODE_ALIGNMENT (V4SFmode);
  addr = choose_baseaddr (frame.stack_realign_offset
			  + xlogue.get_stub_ptr_offset (), &align, AX_REG);
  gcc_assert (align >= GET_MODE_ALIGNMENT (V4SFmode));

  emit_insn (gen_rtx_SET (rax, addr));

  /* Get the stub symbol.  */
  sym = xlogue.get_stub_rtx (frame_pointer_needed ? XLOGUE_STUB_SAVE_HFP
						  : XLOGUE_STUB_SAVE);
  RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);

  for (i = 0; i < ncregs; ++i)
    {
      const xlogue_layout::reginfo &r = xlogue.get_reginfo (i);
      rtx reg = gen_rtx_REG ((SSE_REGNO_P (r.regno) ? V4SFmode : word_mode),
			     r.regno);
      RTVEC_ELT (v, vi++) = gen_frame_store (reg, rax, -r.offset);
    }

  gcc_assert (vi == (unsigned)GET_NUM_ELEM (v));

  insn = emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
  RTX_FRAME_RELATED_P (insn) = true;
}

/* Generate and return an insn body to AND X with Y.  */

static rtx_insn *
gen_and2_insn (rtx x, rtx y)
{
  enum insn_code icode = optab_handler (and_optab, GET_MODE (x));

  gcc_assert (insn_operand_matches (icode, 0, x));
  gcc_assert (insn_operand_matches (icode, 1, x));
  gcc_assert (insn_operand_matches (icode, 2, y));

  return GEN_FCN (icode) (x, x, y);
}

/* Expand the prologue into a bunch of separate insns.  */

void
ix86_expand_prologue (void)
{
  struct machine_function *m = cfun->machine;
  rtx insn, t;
  HOST_WIDE_INT allocate;
  bool int_registers_saved;
  bool sse_registers_saved;
  bool save_stub_call_needed;
  rtx static_chain = NULL_RTX;

  ix86_last_zero_store_uid = 0;
  if (ix86_function_naked (current_function_decl))
    {
      if (flag_stack_usage_info)
	current_function_static_stack_size = 0;
      return;
    }

  ix86_finalize_stack_frame_flags ();

  /* DRAP should not coexist with stack_realign_fp */
  gcc_assert (!(crtl->drap_reg && stack_realign_fp));

  memset (&m->fs, 0, sizeof (m->fs));

  /* Initialize CFA state for before the prologue.  */
  m->fs.cfa_reg = stack_pointer_rtx;
  m->fs.cfa_offset = INCOMING_FRAME_SP_OFFSET;

  /* Track SP offset to the CFA.  We continue tracking this after we've
     swapped the CFA register away from SP.  In the case of re-alignment
     this is fudged; we're interested to offsets within the local frame.  */
  m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
  m->fs.sp_valid = true;
  m->fs.sp_realigned = false;

  const struct ix86_frame &frame = cfun->machine->frame;

  if (!TARGET_64BIT && ix86_function_ms_hook_prologue (current_function_decl))
    {
      /* We should have already generated an error for any use of
         ms_hook on a nested function.  */
      gcc_checking_assert (!ix86_static_chain_on_stack);

      /* Check if profiling is active and we shall use profiling before
         prologue variant. If so sorry.  */
      if (crtl->profile && flag_fentry != 0)
	sorry ("%<ms_hook_prologue%> attribute is not compatible "
	       "with %<-mfentry%> for 32-bit");

      /* In ix86_asm_output_function_label we emitted:
	 8b ff     movl.s %edi,%edi
	 55        push   %ebp
	 8b ec     movl.s %esp,%ebp

	 This matches the hookable function prologue in Win32 API
	 functions in Microsoft Windows XP Service Pack 2 and newer.
	 Wine uses this to enable Windows apps to hook the Win32 API
	 functions provided by Wine.

	 What that means is that we've already set up the frame pointer.  */

      if (frame_pointer_needed
	  && !(crtl->drap_reg && crtl->stack_realign_needed))
	{
	  rtx push, mov;

	  /* We've decided to use the frame pointer already set up.
	     Describe this to the unwinder by pretending that both
	     push and mov insns happen right here.

	     Putting the unwind info here at the end of the ms_hook
	     is done so that we can make absolutely certain we get
	     the required byte sequence at the start of the function,
	     rather than relying on an assembler that can produce
	     the exact encoding required.

	     However it does mean (in the unpatched case) that we have
	     a 1 insn window where the asynchronous unwind info is
	     incorrect.  However, if we placed the unwind info at
	     its correct location we would have incorrect unwind info
	     in the patched case.  Which is probably all moot since
	     I don't expect Wine generates dwarf2 unwind info for the
	     system libraries that use this feature.  */

	  insn = emit_insn (gen_blockage ());

	  push = gen_push (hard_frame_pointer_rtx);
	  mov = gen_rtx_SET (hard_frame_pointer_rtx,
			     stack_pointer_rtx);
	  RTX_FRAME_RELATED_P (push) = 1;
	  RTX_FRAME_RELATED_P (mov) = 1;

	  RTX_FRAME_RELATED_P (insn) = 1;
	  add_reg_note (insn, REG_FRAME_RELATED_EXPR,
			gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, push, mov)));

	  /* Note that gen_push incremented m->fs.cfa_offset, even
	     though we didn't emit the push insn here.  */
	  m->fs.cfa_reg = hard_frame_pointer_rtx;
	  m->fs.fp_offset = m->fs.cfa_offset;
	  m->fs.fp_valid = true;
	}
      else
	{
	  /* The frame pointer is not needed so pop %ebp again.
	     This leaves us with a pristine state.  */
	  emit_insn (gen_pop (hard_frame_pointer_rtx));
	}
    }

  /* The first insn of a function that accepts its static chain on the
     stack is to push the register that would be filled in by a direct
     call.  This insn will be skipped by the trampoline.  */
  else if (ix86_static_chain_on_stack)
    {
      static_chain = ix86_static_chain (cfun->decl, false);
      insn = emit_insn (gen_push (static_chain));
      emit_insn (gen_blockage ());

      /* We don't want to interpret this push insn as a register save,
	 only as a stack adjustment.  The real copy of the register as
	 a save will be done later, if needed.  */
      t = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
      t = gen_rtx_SET (stack_pointer_rtx, t);
      add_reg_note (insn, REG_CFA_ADJUST_CFA, t);
      RTX_FRAME_RELATED_P (insn) = 1;
    }

  /* Emit prologue code to adjust stack alignment and setup DRAP, in case
     of DRAP is needed and stack realignment is really needed after reload */
  if (stack_realign_drap)
    {
      int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;

      /* Can't use DRAP in interrupt function.  */
      if (cfun->machine->func_type != TYPE_NORMAL)
	sorry ("Dynamic Realign Argument Pointer (DRAP) not supported "
	       "in interrupt service routine.  This may be worked "
	       "around by avoiding functions with aggregate return.");

      /* Only need to push parameter pointer reg if it is caller saved.  */
      if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
	{
	  /* Push arg pointer reg */
	  insn = emit_insn (gen_push (crtl->drap_reg));
	  RTX_FRAME_RELATED_P (insn) = 1;
	}

      /* Grab the argument pointer.  */
      t = plus_constant (Pmode, stack_pointer_rtx, m->fs.sp_offset);
      insn = emit_insn (gen_rtx_SET (crtl->drap_reg, t));
      RTX_FRAME_RELATED_P (insn) = 1;
      m->fs.cfa_reg = crtl->drap_reg;
      m->fs.cfa_offset = 0;

      /* Align the stack.  */
      insn = emit_insn (gen_and2_insn (stack_pointer_rtx,
				       GEN_INT (-align_bytes)));
      RTX_FRAME_RELATED_P (insn) = 1;

      /* Replicate the return address on the stack so that return
	 address can be reached via (argp - 1) slot.  This is needed
	 to implement macro RETURN_ADDR_RTX and intrinsic function
	 expand_builtin_return_addr etc.  */
      t = plus_constant (Pmode, crtl->drap_reg, -UNITS_PER_WORD);
      t = gen_frame_mem (word_mode, t);
      insn = emit_insn (gen_push (t));
      RTX_FRAME_RELATED_P (insn) = 1;

      /* For the purposes of frame and register save area addressing,
	 we've started over with a new frame.  */
      m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
      m->fs.realigned = true;

      if (static_chain)
	{
	  /* Replicate static chain on the stack so that static chain
	     can be reached via (argp - 2) slot.  This is needed for
	     nested function with stack realignment.  */
	  insn = emit_insn (gen_push (static_chain));
	  RTX_FRAME_RELATED_P (insn) = 1;
	}
    }

  int_registers_saved = (frame.nregs == 0);
  sse_registers_saved = (frame.nsseregs == 0);
  save_stub_call_needed = (m->call_ms2sysv);
  gcc_assert (sse_registers_saved || !save_stub_call_needed);

  if (frame_pointer_needed && !m->fs.fp_valid)
    {
      /* Note: AT&T enter does NOT have reversed args.  Enter is probably
         slower on all targets.  Also sdb didn't like it.  */
      insn = emit_insn (gen_push (hard_frame_pointer_rtx));
      RTX_FRAME_RELATED_P (insn) = 1;

      if (m->fs.sp_offset == frame.hard_frame_pointer_offset)
	{
	  insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
	  RTX_FRAME_RELATED_P (insn) = 1;

	  if (m->fs.cfa_reg == stack_pointer_rtx)
	    m->fs.cfa_reg = hard_frame_pointer_rtx;
	  m->fs.fp_offset = m->fs.sp_offset;
	  m->fs.fp_valid = true;
	}
    }

  if (!int_registers_saved)
    {
      /* If saving registers via PUSH, do so now.  */
      if (!frame.save_regs_using_mov)
	{
	  ix86_emit_save_regs ();
	  int_registers_saved = true;
	  gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
	}

      /* When using red zone we may start register saving before allocating
	 the stack frame saving one cycle of the prologue.  However, avoid
	 doing this if we have to probe the stack; at least on x86_64 the
	 stack probe can turn into a call that clobbers a red zone location. */
      else if (ix86_using_red_zone ()
	       && (! TARGET_STACK_PROBE
		   || frame.stack_pointer_offset < CHECK_STACK_LIMIT))
	{
	  ix86_emit_save_regs_using_mov (frame.reg_save_offset);
	  cfun->machine->red_zone_used = true;
	  int_registers_saved = true;
	}
    }

  if (frame.red_zone_size != 0)
    cfun->machine->red_zone_used = true;

  if (stack_realign_fp)
    {
      int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
      gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT);

      /* Record last valid frame pointer offset.  */
      m->fs.sp_realigned_fp_last = frame.reg_save_offset;

      /* The computation of the size of the re-aligned stack frame means
	 that we must allocate the size of the register save area before
	 performing the actual alignment.  Otherwise we cannot guarantee
	 that there's enough storage above the realignment point.  */
      allocate = frame.reg_save_offset - m->fs.sp_offset
		 + frame.stack_realign_allocate;
      if (allocate)
        pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				   GEN_INT (-allocate), -1, false);

      /* Align the stack.  */
      emit_insn (gen_and2_insn (stack_pointer_rtx, GEN_INT (-align_bytes)));
      m->fs.sp_offset = ROUND_UP (m->fs.sp_offset, align_bytes);
      m->fs.sp_realigned_offset = m->fs.sp_offset
					      - frame.stack_realign_allocate;
      /* The stack pointer may no longer be equal to CFA - m->fs.sp_offset.
	 Beyond this point, stack access should be done via choose_baseaddr or
	 by using sp_valid_at and fp_valid_at to determine the correct base
	 register.  Henceforth, any CFA offset should be thought of as logical
	 and not physical.  */
      gcc_assert (m->fs.sp_realigned_offset >= m->fs.sp_realigned_fp_last);
      gcc_assert (m->fs.sp_realigned_offset == frame.stack_realign_offset);
      m->fs.sp_realigned = true;

      /* SEH unwind emit doesn't currently support REG_CFA_EXPRESSION, which
	 is needed to describe where a register is saved using a realigned
	 stack pointer, so we need to invalidate the stack pointer for that
	 target.  */
      if (TARGET_SEH)
	m->fs.sp_valid = false;

      /* If SP offset is non-immediate after allocation of the stack frame,
	 then emit SSE saves or stub call prior to allocating the rest of the
	 stack frame.  This is less efficient for the out-of-line stub because
	 we can't combine allocations across the call barrier, but it's better
	 than using a scratch register.  */
      else if (!x86_64_immediate_operand (GEN_INT (frame.stack_pointer_offset
						   - m->fs.sp_realigned_offset),
					  Pmode))
	{
	  if (!sse_registers_saved)
	    {
	      ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
	      sse_registers_saved = true;
	    }
	  else if (save_stub_call_needed)
	    {
	      ix86_emit_outlined_ms2sysv_save (frame);
	      save_stub_call_needed = false;
	    }
	}
    }

  allocate = frame.stack_pointer_offset - m->fs.sp_offset;

  if (flag_stack_usage_info)
    {
      /* We start to count from ARG_POINTER.  */
      HOST_WIDE_INT stack_size = frame.stack_pointer_offset;

      /* If it was realigned, take into account the fake frame.  */
      if (stack_realign_drap)
	{
	  if (ix86_static_chain_on_stack)
	    stack_size += UNITS_PER_WORD;

	  if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
	    stack_size += UNITS_PER_WORD;

	  /* This over-estimates by 1 minimal-stack-alignment-unit but
	     mitigates that by counting in the new return address slot.  */
	  current_function_dynamic_stack_size
	    += crtl->stack_alignment_needed / BITS_PER_UNIT;
	}

      current_function_static_stack_size = stack_size;
    }

  /* On SEH target with very large frame size, allocate an area to save
     SSE registers (as the very large allocation won't be described).  */
  if (TARGET_SEH
      && frame.stack_pointer_offset > SEH_MAX_FRAME_SIZE
      && !sse_registers_saved)
    {
      HOST_WIDE_INT sse_size
	= frame.sse_reg_save_offset - frame.reg_save_offset;

      gcc_assert (int_registers_saved);

      /* No need to do stack checking as the area will be immediately
	 written.  */
      pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
			         GEN_INT (-sse_size), -1,
				 m->fs.cfa_reg == stack_pointer_rtx);
      allocate -= sse_size;
      ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
      sse_registers_saved = true;
    }

  /* If stack clash protection is requested, then probe the stack, unless it
     is already probed on the target.  */
  if (allocate >= 0
      && flag_stack_clash_protection
      && !ix86_target_stack_probe ())
    {
      ix86_adjust_stack_and_probe (allocate, int_registers_saved, false);
      allocate = 0;
    }

  /* The stack has already been decremented by the instruction calling us
     so probe if the size is non-negative to preserve the protection area.  */
  else if (allocate >= 0 && flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
    {
      const HOST_WIDE_INT probe_interval = get_probe_interval ();

      if (STACK_CHECK_MOVING_SP)
	{
	  if (crtl->is_leaf
	      && !cfun->calls_alloca
	      && allocate <= probe_interval)
	    ;

	  else
	    {
	      ix86_adjust_stack_and_probe (allocate, int_registers_saved, true);
	      allocate = 0;
	    }
	}

      else
	{
	  HOST_WIDE_INT size = allocate;

	  if (TARGET_64BIT && size >= HOST_WIDE_INT_C (0x80000000))
	    size = 0x80000000 - get_stack_check_protect () - 1;

	  if (TARGET_STACK_PROBE)
	    {
	      if (crtl->is_leaf && !cfun->calls_alloca)
		{
		  if (size > probe_interval)
		    ix86_emit_probe_stack_range (0, size, int_registers_saved);
		}
	      else
		ix86_emit_probe_stack_range (0,
					     size + get_stack_check_protect (),
					     int_registers_saved);
	    }
	  else
	    {
	      if (crtl->is_leaf && !cfun->calls_alloca)
		{
		  if (size > probe_interval
		      && size > get_stack_check_protect ())
		    ix86_emit_probe_stack_range (get_stack_check_protect (),
						 (size
						  - get_stack_check_protect ()),
						 int_registers_saved);
		}
	      else
		ix86_emit_probe_stack_range (get_stack_check_protect (), size,
					     int_registers_saved);
	    }
	}
    }

  if (allocate == 0)
    ;
  else if (!ix86_target_stack_probe ()
	   || frame.stack_pointer_offset < CHECK_STACK_LIMIT)
    {
      pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
			         GEN_INT (-allocate), -1,
			         m->fs.cfa_reg == stack_pointer_rtx);
    }
  else
    {
      rtx eax = gen_rtx_REG (Pmode, AX_REG);
      rtx r10 = NULL;
      const bool sp_is_cfa_reg = (m->fs.cfa_reg == stack_pointer_rtx);
      bool eax_live = ix86_eax_live_at_start_p ();
      bool r10_live = false;

      if (TARGET_64BIT)
        r10_live = (DECL_STATIC_CHAIN (current_function_decl) != 0);

      if (eax_live)
	{
	  insn = emit_insn (gen_push (eax));
	  allocate -= UNITS_PER_WORD;
	  /* Note that SEH directives need to continue tracking the stack
	     pointer even after the frame pointer has been set up.  */
	  if (sp_is_cfa_reg || TARGET_SEH)
	    {
	      if (sp_is_cfa_reg)
		m->fs.cfa_offset += UNITS_PER_WORD;
	      RTX_FRAME_RELATED_P (insn) = 1;
	      add_reg_note (insn, REG_FRAME_RELATED_EXPR,
			    gen_rtx_SET (stack_pointer_rtx,
					 plus_constant (Pmode,
							stack_pointer_rtx,
							-UNITS_PER_WORD)));
	    }
	}

      if (r10_live)
	{
	  r10 = gen_rtx_REG (Pmode, R10_REG);
	  insn = emit_insn (gen_push (r10));
	  allocate -= UNITS_PER_WORD;
	  if (sp_is_cfa_reg || TARGET_SEH)
	    {
	      if (sp_is_cfa_reg)
		m->fs.cfa_offset += UNITS_PER_WORD;
	      RTX_FRAME_RELATED_P (insn) = 1;
	      add_reg_note (insn, REG_FRAME_RELATED_EXPR,
			    gen_rtx_SET (stack_pointer_rtx,
					 plus_constant (Pmode,
							stack_pointer_rtx,
							-UNITS_PER_WORD)));
	    }
	}

      emit_move_insn (eax, GEN_INT (allocate));
      emit_insn (gen_allocate_stack_worker_probe (Pmode, eax, eax));

      /* Use the fact that AX still contains ALLOCATE.  */
      insn = emit_insn (gen_pro_epilogue_adjust_stack_sub
			(Pmode, stack_pointer_rtx, stack_pointer_rtx, eax));

      if (sp_is_cfa_reg || TARGET_SEH)
	{
	  if (sp_is_cfa_reg)
	    m->fs.cfa_offset += allocate;
	  RTX_FRAME_RELATED_P (insn) = 1;
	  add_reg_note (insn, REG_FRAME_RELATED_EXPR,
			gen_rtx_SET (stack_pointer_rtx,
				     plus_constant (Pmode, stack_pointer_rtx,
						    -allocate)));
	}
      m->fs.sp_offset += allocate;

      /* Use stack_pointer_rtx for relative addressing so that code works for
	 realigned stack.  But this means that we need a blockage to prevent
	 stores based on the frame pointer from being scheduled before.  */
      if (r10_live && eax_live)
        {
	  t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
	  emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
			  gen_frame_mem (word_mode, t));
	  t = plus_constant (Pmode, t, UNITS_PER_WORD);
	  emit_move_insn (gen_rtx_REG (word_mode, AX_REG),
			  gen_frame_mem (word_mode, t));
	  emit_insn (gen_memory_blockage ());
	}
      else if (eax_live || r10_live)
	{
	  t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
	  emit_move_insn (gen_rtx_REG (word_mode,
				       (eax_live ? AX_REG : R10_REG)),
			  gen_frame_mem (word_mode, t));
	  emit_insn (gen_memory_blockage ());
	}
    }
  gcc_assert (m->fs.sp_offset == frame.stack_pointer_offset);

  /* If we havn't already set up the frame pointer, do so now.  */
  if (frame_pointer_needed && !m->fs.fp_valid)
    {
      insn = gen_add3_insn (hard_frame_pointer_rtx, stack_pointer_rtx,
			    GEN_INT (frame.stack_pointer_offset
				     - frame.hard_frame_pointer_offset));
      insn = emit_insn (insn);
      RTX_FRAME_RELATED_P (insn) = 1;
      add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);

      if (m->fs.cfa_reg == stack_pointer_rtx)
	m->fs.cfa_reg = hard_frame_pointer_rtx;
      m->fs.fp_offset = frame.hard_frame_pointer_offset;
      m->fs.fp_valid = true;
    }

  if (!int_registers_saved)
    ix86_emit_save_regs_using_mov (frame.reg_save_offset);
  if (!sse_registers_saved)
    ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
  else if (save_stub_call_needed)
    ix86_emit_outlined_ms2sysv_save (frame);

  /* For the mcount profiling on 32 bit PIC mode we need to emit SET_GOT
     in PROLOGUE.  */
  if (!TARGET_64BIT && pic_offset_table_rtx && crtl->profile && !flag_fentry)
    {
      rtx pic = gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM);
      insn = emit_insn (gen_set_got (pic));
      RTX_FRAME_RELATED_P (insn) = 1;
      add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
      emit_insn (gen_prologue_use (pic));
      /* Deleting already emmitted SET_GOT if exist and allocated to
	 REAL_PIC_OFFSET_TABLE_REGNUM.  */
      ix86_elim_entry_set_got (pic);
    }

  if (crtl->drap_reg && !crtl->stack_realign_needed)
    {
      /* vDRAP is setup but after reload it turns out stack realign
         isn't necessary, here we will emit prologue to setup DRAP
         without stack realign adjustment */
      t = choose_baseaddr (0, NULL);
      emit_insn (gen_rtx_SET (crtl->drap_reg, t));
    }

  /* Prevent instructions from being scheduled into register save push
     sequence when access to the redzone area is done through frame pointer.
     The offset between the frame pointer and the stack pointer is calculated
     relative to the value of the stack pointer at the end of the function
     prologue, and moving instructions that access redzone area via frame
     pointer inside push sequence violates this assumption.  */
  if (frame_pointer_needed && frame.red_zone_size)
    emit_insn (gen_memory_blockage ());

  /* SEH requires that the prologue end within 256 bytes of the start of
     the function.  Prevent instruction schedules that would extend that.
     Further, prevent alloca modifications to the stack pointer from being
     combined with prologue modifications.  */
  if (TARGET_SEH)
    emit_insn (gen_prologue_use (stack_pointer_rtx));
}

/* Emit code to restore REG using a POP insn.  */

static void
ix86_emit_restore_reg_using_pop (rtx reg)
{
  struct machine_function *m = cfun->machine;
  rtx_insn *insn = emit_insn (gen_pop (reg));

  ix86_add_cfa_restore_note (insn, reg, m->fs.sp_offset);
  m->fs.sp_offset -= UNITS_PER_WORD;

  if (m->fs.cfa_reg == crtl->drap_reg
      && REGNO (reg) == REGNO (crtl->drap_reg))
    {
      /* Previously we'd represented the CFA as an expression
	 like *(%ebp - 8).  We've just popped that value from
	 the stack, which means we need to reset the CFA to
	 the drap register.  This will remain until we restore
	 the stack pointer.  */
      add_reg_note (insn, REG_CFA_DEF_CFA, reg);
      RTX_FRAME_RELATED_P (insn) = 1;

      /* This means that the DRAP register is valid for addressing too.  */
      m->fs.drap_valid = true;
      return;
    }

  if (m->fs.cfa_reg == stack_pointer_rtx)
    {
      rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
      x = gen_rtx_SET (stack_pointer_rtx, x);
      add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
      RTX_FRAME_RELATED_P (insn) = 1;

      m->fs.cfa_offset -= UNITS_PER_WORD;
    }

  /* When the frame pointer is the CFA, and we pop it, we are
     swapping back to the stack pointer as the CFA.  This happens
     for stack frames that don't allocate other data, so we assume
     the stack pointer is now pointing at the return address, i.e.
     the function entry state, which makes the offset be 1 word.  */
  if (reg == hard_frame_pointer_rtx)
    {
      m->fs.fp_valid = false;
      if (m->fs.cfa_reg == hard_frame_pointer_rtx)
	{
	  m->fs.cfa_reg = stack_pointer_rtx;
	  m->fs.cfa_offset -= UNITS_PER_WORD;

	  add_reg_note (insn, REG_CFA_DEF_CFA,
			plus_constant (Pmode, stack_pointer_rtx,
				       m->fs.cfa_offset));
	  RTX_FRAME_RELATED_P (insn) = 1;
	}
    }
}

/* Emit code to restore saved registers using POP insns.  */

static void
ix86_emit_restore_regs_using_pop (void)
{
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, false, true))
      ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno));
}

/* Emit code and notes for the LEAVE instruction.  If insn is non-null,
   omits the emit and only attaches the notes.  */

static void
ix86_emit_leave (rtx_insn *insn)
{
  struct machine_function *m = cfun->machine;

  if (!insn)
    insn = emit_insn (gen_leave (word_mode));

  ix86_add_queued_cfa_restore_notes (insn);

  gcc_assert (m->fs.fp_valid);
  m->fs.sp_valid = true;
  m->fs.sp_realigned = false;
  m->fs.sp_offset = m->fs.fp_offset - UNITS_PER_WORD;
  m->fs.fp_valid = false;

  if (m->fs.cfa_reg == hard_frame_pointer_rtx)
    {
      m->fs.cfa_reg = stack_pointer_rtx;
      m->fs.cfa_offset = m->fs.sp_offset;

      add_reg_note (insn, REG_CFA_DEF_CFA,
		    plus_constant (Pmode, stack_pointer_rtx,
				   m->fs.sp_offset));
      RTX_FRAME_RELATED_P (insn) = 1;
    }
  ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx,
			     m->fs.fp_offset);
}

/* Emit code to restore saved registers using MOV insns.
   First register is restored from CFA - CFA_OFFSET.  */
static void
ix86_emit_restore_regs_using_mov (HOST_WIDE_INT cfa_offset,
				  bool maybe_eh_return)
{
  struct machine_function *m = cfun->machine;
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return, true))
      {
	rtx reg = gen_rtx_REG (word_mode, regno);
	rtx mem;
	rtx_insn *insn;

	mem = choose_baseaddr (cfa_offset, NULL);
	mem = gen_frame_mem (word_mode, mem);
	insn = emit_move_insn (reg, mem);

        if (m->fs.cfa_reg == crtl->drap_reg && regno == REGNO (crtl->drap_reg))
	  {
	    /* Previously we'd represented the CFA as an expression
	       like *(%ebp - 8).  We've just popped that value from
	       the stack, which means we need to reset the CFA to
	       the drap register.  This will remain until we restore
	       the stack pointer.  */
	    add_reg_note (insn, REG_CFA_DEF_CFA, reg);
	    RTX_FRAME_RELATED_P (insn) = 1;

	    /* This means that the DRAP register is valid for addressing.  */
	    m->fs.drap_valid = true;
	  }
	else
	  ix86_add_cfa_restore_note (NULL, reg, cfa_offset);

	cfa_offset -= UNITS_PER_WORD;
      }
}

/* Emit code to restore saved registers using MOV insns.
   First register is restored from CFA - CFA_OFFSET.  */
static void
ix86_emit_restore_sse_regs_using_mov (HOST_WIDE_INT cfa_offset,
				      bool maybe_eh_return)
{
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return, true))
      {
	rtx reg = gen_rtx_REG (V4SFmode, regno);
	rtx mem;
	unsigned int align = GET_MODE_ALIGNMENT (V4SFmode);

	mem = choose_baseaddr (cfa_offset, &align);
	mem = gen_rtx_MEM (V4SFmode, mem);

	/* The location aligment depends upon the base register.  */
	align = MIN (GET_MODE_ALIGNMENT (V4SFmode), align);
	gcc_assert (! (cfa_offset & (align / BITS_PER_UNIT - 1)));
	set_mem_align (mem, align);
	emit_insn (gen_rtx_SET (reg, mem));

	ix86_add_cfa_restore_note (NULL, reg, cfa_offset);

	cfa_offset -= GET_MODE_SIZE (V4SFmode);
      }
}

static void
ix86_emit_outlined_ms2sysv_restore (const struct ix86_frame &frame,
				  bool use_call, int style)
{
  struct machine_function *m = cfun->machine;
  const unsigned ncregs = NUM_X86_64_MS_CLOBBERED_REGS
			  + m->call_ms2sysv_extra_regs;
  rtvec v;
  unsigned int elems_needed, align, i, vi = 0;
  rtx_insn *insn;
  rtx sym, tmp;
  rtx rsi = gen_rtx_REG (word_mode, SI_REG);
  rtx r10 = NULL_RTX;
  const class xlogue_layout &xlogue = xlogue_layout::get_instance ();
  HOST_WIDE_INT stub_ptr_offset = xlogue.get_stub_ptr_offset ();
  HOST_WIDE_INT rsi_offset = frame.stack_realign_offset + stub_ptr_offset;
  rtx rsi_frame_load = NULL_RTX;
  HOST_WIDE_INT rsi_restore_offset = (HOST_WIDE_INT)-1;
  enum xlogue_stub stub;

  gcc_assert (!m->fs.fp_valid || frame_pointer_needed);

  /* If using a realigned stack, we should never start with padding.  */
  gcc_assert (!stack_realign_fp || !xlogue.get_stack_align_off_in ());

  /* Setup RSI as the stub's base pointer.  */
  align = GET_MODE_ALIGNMENT (V4SFmode);
  tmp = choose_baseaddr (rsi_offset, &align, SI_REG);
  gcc_assert (align >= GET_MODE_ALIGNMENT (V4SFmode));

  emit_insn (gen_rtx_SET (rsi, tmp));

  /* Get a symbol for the stub.  */
  if (frame_pointer_needed)
    stub = use_call ? XLOGUE_STUB_RESTORE_HFP
		    : XLOGUE_STUB_RESTORE_HFP_TAIL;
  else
    stub = use_call ? XLOGUE_STUB_RESTORE
		    : XLOGUE_STUB_RESTORE_TAIL;
  sym = xlogue.get_stub_rtx (stub);

  elems_needed = ncregs;
  if (use_call)
    elems_needed += 1;
  else
    elems_needed += frame_pointer_needed ? 5 : 3;
  v = rtvec_alloc (elems_needed);

  /* We call the epilogue stub when we need to pop incoming args or we are
     doing a sibling call as the tail.  Otherwise, we will emit a jmp to the
     epilogue stub and it is the tail-call.  */
  if (use_call)
      RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);
  else
    {
      RTVEC_ELT (v, vi++) = ret_rtx;
      RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);
      if (frame_pointer_needed)
	{
	  rtx rbp = gen_rtx_REG (DImode, BP_REG);
	  gcc_assert (m->fs.fp_valid);
	  gcc_assert (m->fs.cfa_reg == hard_frame_pointer_rtx);

	  tmp = plus_constant (DImode, rbp, 8);
	  RTVEC_ELT (v, vi++) = gen_rtx_SET (stack_pointer_rtx, tmp);
	  RTVEC_ELT (v, vi++) = gen_rtx_SET (rbp, gen_rtx_MEM (DImode, rbp));
	  tmp = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
	  RTVEC_ELT (v, vi++) = gen_rtx_CLOBBER (VOIDmode, tmp);
	}
      else
	{
	  /* If no hard frame pointer, we set R10 to the SP restore value.  */
	  gcc_assert (!m->fs.fp_valid);
	  gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
	  gcc_assert (m->fs.sp_valid);

	  r10 = gen_rtx_REG (DImode, R10_REG);
	  tmp = plus_constant (Pmode, rsi, stub_ptr_offset);
	  emit_insn (gen_rtx_SET (r10, tmp));

	  RTVEC_ELT (v, vi++) = gen_rtx_SET (stack_pointer_rtx, r10);
	}
    }

  /* Generate frame load insns and restore notes.  */
  for (i = 0; i < ncregs; ++i)
    {
      const xlogue_layout::reginfo &r = xlogue.get_reginfo (i);
      machine_mode mode = SSE_REGNO_P (r.regno) ? V4SFmode : word_mode;
      rtx reg, frame_load;

      reg = gen_rtx_REG (mode, r.regno);
      frame_load = gen_frame_load (reg, rsi, r.offset);

      /* Save RSI frame load insn & note to add last.  */
      if (r.regno == SI_REG)
	{
	  gcc_assert (!rsi_frame_load);
	  rsi_frame_load = frame_load;
	  rsi_restore_offset = r.offset;
	}
      else
	{
	  RTVEC_ELT (v, vi++) = frame_load;
	  ix86_add_cfa_restore_note (NULL, reg, r.offset);
	}
    }

  /* Add RSI frame load & restore note at the end.  */
  gcc_assert (rsi_frame_load);
  gcc_assert (rsi_restore_offset != (HOST_WIDE_INT)-1);
  RTVEC_ELT (v, vi++) = rsi_frame_load;
  ix86_add_cfa_restore_note (NULL, gen_rtx_REG (DImode, SI_REG),
			     rsi_restore_offset);

  /* Finally, for tail-call w/o a hard frame pointer, set SP to R10.  */
  if (!use_call && !frame_pointer_needed)
    {
      gcc_assert (m->fs.sp_valid);
      gcc_assert (!m->fs.sp_realigned);

      /* At this point, R10 should point to frame.stack_realign_offset.  */
      if (m->fs.cfa_reg == stack_pointer_rtx)
	m->fs.cfa_offset += m->fs.sp_offset - frame.stack_realign_offset;
      m->fs.sp_offset = frame.stack_realign_offset;
    }

  gcc_assert (vi == (unsigned int)GET_NUM_ELEM (v));
  tmp = gen_rtx_PARALLEL (VOIDmode, v);
  if (use_call)
      insn = emit_insn (tmp);
  else
    {
      insn = emit_jump_insn (tmp);
      JUMP_LABEL (insn) = ret_rtx;

      if (frame_pointer_needed)
	ix86_emit_leave (insn);
      else
	{
	  /* Need CFA adjust note.  */
	  tmp = gen_rtx_SET (stack_pointer_rtx, r10);
	  add_reg_note (insn, REG_CFA_ADJUST_CFA, tmp);
	}
    }

  RTX_FRAME_RELATED_P (insn) = true;
  ix86_add_queued_cfa_restore_notes (insn);

  /* If we're not doing a tail-call, we need to adjust the stack.  */
  if (use_call && m->fs.sp_valid)
    {
      HOST_WIDE_INT dealloc = m->fs.sp_offset - frame.stack_realign_offset;
      pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				GEN_INT (dealloc), style,
				m->fs.cfa_reg == stack_pointer_rtx);
    }
}

/* Restore function stack, frame, and registers.  */

void
ix86_expand_epilogue (int style)
{
  struct machine_function *m = cfun->machine;
  struct machine_frame_state frame_state_save = m->fs;
  bool restore_regs_via_mov;
  bool using_drap;
  bool restore_stub_is_tail = false;

  if (ix86_function_naked (current_function_decl))
    {
      /* The program should not reach this point.  */
      emit_insn (gen_ud2 ());
      return;
    }

  ix86_finalize_stack_frame_flags ();
  const struct ix86_frame &frame = cfun->machine->frame;

  m->fs.sp_realigned = stack_realign_fp;
  m->fs.sp_valid = stack_realign_fp
		   || !frame_pointer_needed
		   || crtl->sp_is_unchanging;
  gcc_assert (!m->fs.sp_valid
	      || m->fs.sp_offset == frame.stack_pointer_offset);

  /* The FP must be valid if the frame pointer is present.  */
  gcc_assert (frame_pointer_needed == m->fs.fp_valid);
  gcc_assert (!m->fs.fp_valid
	      || m->fs.fp_offset == frame.hard_frame_pointer_offset);

  /* We must have *some* valid pointer to the stack frame.  */
  gcc_assert (m->fs.sp_valid || m->fs.fp_valid);

  /* The DRAP is never valid at this point.  */
  gcc_assert (!m->fs.drap_valid);

  /* See the comment about red zone and frame
     pointer usage in ix86_expand_prologue.  */
  if (frame_pointer_needed && frame.red_zone_size)
    emit_insn (gen_memory_blockage ());

  using_drap = crtl->drap_reg && crtl->stack_realign_needed;
  gcc_assert (!using_drap || m->fs.cfa_reg == crtl->drap_reg);

  /* Determine the CFA offset of the end of the red-zone.  */
  m->fs.red_zone_offset = 0;
  if (ix86_using_red_zone () && crtl->args.pops_args < 65536)
    {
      /* The red-zone begins below return address and error code in
	 exception handler.  */
      m->fs.red_zone_offset = RED_ZONE_SIZE + INCOMING_FRAME_SP_OFFSET;

      /* When the register save area is in the aligned portion of
         the stack, determine the maximum runtime displacement that
	 matches up with the aligned frame.  */
      if (stack_realign_drap)
	m->fs.red_zone_offset -= (crtl->stack_alignment_needed / BITS_PER_UNIT
				  + UNITS_PER_WORD);
    }

  HOST_WIDE_INT reg_save_offset = frame.reg_save_offset;

  /* Special care must be taken for the normal return case of a function
     using eh_return: the eax and edx registers are marked as saved, but
     not restored along this path.  Adjust the save location to match.  */
  if (crtl->calls_eh_return && style != 2)
    reg_save_offset -= 2 * UNITS_PER_WORD;

  /* EH_RETURN requires the use of moves to function properly.  */
  if (crtl->calls_eh_return)
    restore_regs_via_mov = true;
  /* SEH requires the use of pops to identify the epilogue.  */
  else if (TARGET_SEH)
    restore_regs_via_mov = false;
  /* If we're only restoring one register and sp cannot be used then
     using a move instruction to restore the register since it's
     less work than reloading sp and popping the register.  */
  else if (!sp_valid_at (frame.hfp_save_offset) && frame.nregs <= 1)
    restore_regs_via_mov = true;
  else if (TARGET_EPILOGUE_USING_MOVE
	   && cfun->machine->use_fast_prologue_epilogue
	   && (frame.nregs > 1
	       || m->fs.sp_offset != reg_save_offset))
    restore_regs_via_mov = true;
  else if (frame_pointer_needed
	   && !frame.nregs
	   && m->fs.sp_offset != reg_save_offset)
    restore_regs_via_mov = true;
  else if (frame_pointer_needed
	   && TARGET_USE_LEAVE
	   && cfun->machine->use_fast_prologue_epilogue
	   && frame.nregs == 1)
    restore_regs_via_mov = true;
  else
    restore_regs_via_mov = false;

  if (restore_regs_via_mov || frame.nsseregs)
    {
      /* Ensure that the entire register save area is addressable via
	 the stack pointer, if we will restore SSE regs via sp.  */
      if (TARGET_64BIT
	  && m->fs.sp_offset > 0x7fffffff
	  && sp_valid_at (frame.stack_realign_offset + 1)
	  && (frame.nsseregs + frame.nregs) != 0)
	{
	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				     GEN_INT (m->fs.sp_offset
					      - frame.sse_reg_save_offset),
				     style,
				     m->fs.cfa_reg == stack_pointer_rtx);
	}
    }

  /* If there are any SSE registers to restore, then we have to do it
     via moves, since there's obviously no pop for SSE regs.  */
  if (frame.nsseregs)
    ix86_emit_restore_sse_regs_using_mov (frame.sse_reg_save_offset,
					  style == 2);

  if (m->call_ms2sysv)
    {
      int pop_incoming_args = crtl->args.pops_args && crtl->args.size;

      /* We cannot use a tail-call for the stub if:
	 1. We have to pop incoming args,
	 2. We have additional int regs to restore, or
	 3. A sibling call will be the tail-call, or
	 4. We are emitting an eh_return_internal epilogue.

	 TODO: Item 4 has not yet tested!

	 If any of the above are true, we will call the stub rather than
	 jump to it.  */
      restore_stub_is_tail = !(pop_incoming_args || frame.nregs || style != 1);
      ix86_emit_outlined_ms2sysv_restore (frame, !restore_stub_is_tail, style);
    }

  /* If using out-of-line stub that is a tail-call, then...*/
  if (m->call_ms2sysv && restore_stub_is_tail)
    {
      /* TODO: parinoid tests. (remove eventually)  */
      gcc_assert (m->fs.sp_valid);
      gcc_assert (!m->fs.sp_realigned);
      gcc_assert (!m->fs.fp_valid);
      gcc_assert (!m->fs.realigned);
      gcc_assert (m->fs.sp_offset == UNITS_PER_WORD);
      gcc_assert (!crtl->drap_reg);
      gcc_assert (!frame.nregs);
    }
  else if (restore_regs_via_mov)
    {
      rtx t;

      if (frame.nregs)
	ix86_emit_restore_regs_using_mov (reg_save_offset, style == 2);

      /* eh_return epilogues need %ecx added to the stack pointer.  */
      if (style == 2)
	{
	  rtx sa = EH_RETURN_STACKADJ_RTX;
	  rtx_insn *insn;

	  /* Stack realignment doesn't work with eh_return.  */
	  if (crtl->stack_realign_needed)
	    sorry ("Stack realignment not supported with "
		   "%<__builtin_eh_return%>");

	  /* regparm nested functions don't work with eh_return.  */
	  if (ix86_static_chain_on_stack)
	    sorry ("regparm nested function not supported with "
		   "%<__builtin_eh_return%>");

	  if (frame_pointer_needed)
	    {
	      t = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
	      t = plus_constant (Pmode, t, m->fs.fp_offset - UNITS_PER_WORD);
	      emit_insn (gen_rtx_SET (sa, t));

	      /* NB: eh_return epilogues must restore the frame pointer
		 in word_mode since the upper 32 bits of RBP register
		 can have any values.  */
	      t = gen_frame_mem (word_mode, hard_frame_pointer_rtx);
	      rtx frame_reg = gen_rtx_REG (word_mode,
					   HARD_FRAME_POINTER_REGNUM);
	      insn = emit_move_insn (frame_reg, t);

	      /* Note that we use SA as a temporary CFA, as the return
		 address is at the proper place relative to it.  We
		 pretend this happens at the FP restore insn because
		 prior to this insn the FP would be stored at the wrong
		 offset relative to SA, and after this insn we have no
		 other reasonable register to use for the CFA.  We don't
		 bother resetting the CFA to the SP for the duration of
		 the return insn, unless the control flow instrumentation
		 is done.  In this case the SP is used later and we have
		 to reset CFA to SP.  */
	      add_reg_note (insn, REG_CFA_DEF_CFA,
			    plus_constant (Pmode, sa, UNITS_PER_WORD));
	      ix86_add_queued_cfa_restore_notes (insn);
	      add_reg_note (insn, REG_CFA_RESTORE, frame_reg);
	      RTX_FRAME_RELATED_P (insn) = 1;

	      m->fs.cfa_reg = sa;
	      m->fs.cfa_offset = UNITS_PER_WORD;
	      m->fs.fp_valid = false;

	      pro_epilogue_adjust_stack (stack_pointer_rtx, sa,
					 const0_rtx, style,
					 flag_cf_protection);
	    }
	  else
	    {
	      t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
	      t = plus_constant (Pmode, t, m->fs.sp_offset - UNITS_PER_WORD);
	      insn = emit_insn (gen_rtx_SET (stack_pointer_rtx, t));
	      ix86_add_queued_cfa_restore_notes (insn);

	      gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
	      if (m->fs.cfa_offset != UNITS_PER_WORD)
		{
		  m->fs.cfa_offset = UNITS_PER_WORD;
		  add_reg_note (insn, REG_CFA_DEF_CFA,
				plus_constant (Pmode, stack_pointer_rtx,
					       UNITS_PER_WORD));
		  RTX_FRAME_RELATED_P (insn) = 1;
		}
	    }
	  m->fs.sp_offset = UNITS_PER_WORD;
	  m->fs.sp_valid = true;
	  m->fs.sp_realigned = false;
	}
    }
  else
    {
      /* SEH requires that the function end with (1) a stack adjustment
	 if necessary, (2) a sequence of pops, and (3) a return or
	 jump instruction.  Prevent insns from the function body from
	 being scheduled into this sequence.  */
      if (TARGET_SEH)
	{
	  /* Prevent a catch region from being adjacent to the standard
	     epilogue sequence.  Unfortunately neither crtl->uses_eh_lsda
	     nor several other flags that would be interesting to test are
	     set up yet.  */
	  if (flag_non_call_exceptions)
	    emit_insn (gen_nops (const1_rtx));
	  else
	    emit_insn (gen_blockage ());
	}

      /* First step is to deallocate the stack frame so that we can
	 pop the registers.  If the stack pointer was realigned, it needs
	 to be restored now.  Also do it on SEH target for very large
	 frame as the emitted instructions aren't allowed by the ABI
	 in epilogues.  */
      if (!m->fs.sp_valid || m->fs.sp_realigned
 	  || (TARGET_SEH
	      && (m->fs.sp_offset - reg_save_offset
		  >= SEH_MAX_FRAME_SIZE)))
	{
	  pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx,
				     GEN_INT (m->fs.fp_offset
					      - reg_save_offset),
				     style, false);
	}
      else if (m->fs.sp_offset != reg_save_offset)
	{
	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				     GEN_INT (m->fs.sp_offset
					      - reg_save_offset),
				     style,
				     m->fs.cfa_reg == stack_pointer_rtx);
	}

      ix86_emit_restore_regs_using_pop ();
    }

  /* If we used a stack pointer and haven't already got rid of it,
     then do so now.  */
  if (m->fs.fp_valid)
    {
      /* If the stack pointer is valid and pointing at the frame
	 pointer store address, then we only need a pop.  */
      if (sp_valid_at (frame.hfp_save_offset)
	  && m->fs.sp_offset == frame.hfp_save_offset)
	ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
      /* Leave results in shorter dependency chains on CPUs that are
	 able to grok it fast.  */
      else if (TARGET_USE_LEAVE
	       || optimize_bb_for_size_p (EXIT_BLOCK_PTR_FOR_FN (cfun))
	       || !cfun->machine->use_fast_prologue_epilogue)
	ix86_emit_leave (NULL);
      else
        {
	  pro_epilogue_adjust_stack (stack_pointer_rtx,
				     hard_frame_pointer_rtx,
				     const0_rtx, style, !using_drap);
	  ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
        }
    }

  if (using_drap)
    {
      int param_ptr_offset = UNITS_PER_WORD;
      rtx_insn *insn;

      gcc_assert (stack_realign_drap);

      if (ix86_static_chain_on_stack)
	param_ptr_offset += UNITS_PER_WORD;
      if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
	param_ptr_offset += UNITS_PER_WORD;

      insn = emit_insn (gen_rtx_SET
			(stack_pointer_rtx,
			 plus_constant (Pmode, crtl->drap_reg,
					-param_ptr_offset)));
      m->fs.cfa_reg = stack_pointer_rtx;
      m->fs.cfa_offset = param_ptr_offset;
      m->fs.sp_offset = param_ptr_offset;
      m->fs.realigned = false;

      add_reg_note (insn, REG_CFA_DEF_CFA,
		    plus_constant (Pmode, stack_pointer_rtx,
				   param_ptr_offset));
      RTX_FRAME_RELATED_P (insn) = 1;

      if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
	ix86_emit_restore_reg_using_pop (crtl->drap_reg);
    }

  /* At this point the stack pointer must be valid, and we must have
     restored all of the registers.  We may not have deallocated the
     entire stack frame.  We've delayed this until now because it may
     be possible to merge the local stack deallocation with the
     deallocation forced by ix86_static_chain_on_stack.   */
  gcc_assert (m->fs.sp_valid);
  gcc_assert (!m->fs.sp_realigned);
  gcc_assert (!m->fs.fp_valid);
  gcc_assert (!m->fs.realigned);
  if (m->fs.sp_offset != UNITS_PER_WORD)
    {
      pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				 GEN_INT (m->fs.sp_offset - UNITS_PER_WORD),
				 style, true);
    }
  else
    ix86_add_queued_cfa_restore_notes (get_last_insn ());

  /* Sibcall epilogues don't want a return instruction.  */
  if (style == 0)
    {
      m->fs = frame_state_save;
      return;
    }

  if (cfun->machine->func_type != TYPE_NORMAL)
    emit_jump_insn (gen_interrupt_return ());
  else if (crtl->args.pops_args && crtl->args.size)
    {
      rtx popc = GEN_INT (crtl->args.pops_args);

      /* i386 can only pop 64K bytes.  If asked to pop more, pop return
	 address, do explicit add, and jump indirectly to the caller.  */

      if (crtl->args.pops_args >= 65536)
	{
	  rtx ecx = gen_rtx_REG (SImode, CX_REG);
	  rtx_insn *insn;

	  /* There is no "pascal" calling convention in any 64bit ABI.  */
	  gcc_assert (!TARGET_64BIT);

	  insn = emit_insn (gen_pop (ecx));
	  m->fs.cfa_offset -= UNITS_PER_WORD;
	  m->fs.sp_offset -= UNITS_PER_WORD;

	  rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
	  x = gen_rtx_SET (stack_pointer_rtx, x);
	  add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
	  add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
	  RTX_FRAME_RELATED_P (insn) = 1;

	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				     popc, -1, true);
	  emit_jump_insn (gen_simple_return_indirect_internal (ecx));
	}
      else
	emit_jump_insn (gen_simple_return_pop_internal (popc));
    }
  else if (!m->call_ms2sysv || !restore_stub_is_tail)
    {
      /* In case of return from EH a simple return cannot be used
	 as a return address will be compared with a shadow stack
	 return address.  Use indirect jump instead.  */
      if (style == 2 && flag_cf_protection)
	{
	  /* Register used in indirect jump must be in word_mode.  But
	     Pmode may not be the same as word_mode for x32.  */
	  rtx ecx = gen_rtx_REG (word_mode, CX_REG);
	  rtx_insn *insn;

	  insn = emit_insn (gen_pop (ecx));
	  m->fs.cfa_offset -= UNITS_PER_WORD;
	  m->fs.sp_offset -= UNITS_PER_WORD;

	  rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
	  x = gen_rtx_SET (stack_pointer_rtx, x);
	  add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
	  add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
	  RTX_FRAME_RELATED_P (insn) = 1;

	  emit_jump_insn (gen_simple_return_indirect_internal (ecx));
	}
      else
	emit_jump_insn (gen_simple_return_internal ());
    }

  /* Restore the state back to the state from the prologue,
     so that it's correct for the next epilogue.  */
  m->fs = frame_state_save;
}

/* Reset from the function's potential modifications.  */

static void
ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED)
{
  if (pic_offset_table_rtx
      && !ix86_use_pseudo_pic_reg ())
    SET_REGNO (pic_offset_table_rtx, REAL_PIC_OFFSET_TABLE_REGNUM);

  if (TARGET_MACHO)
    {
      rtx_insn *insn = get_last_insn ();
      rtx_insn *deleted_debug_label = NULL;

      /* Mach-O doesn't support labels at the end of objects, so if
         it looks like we might want one, take special action.
        First, collect any sequence of deleted debug labels.  */
      while (insn
	     && NOTE_P (insn)
	     && NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
	{
	  /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
	     notes only, instead set their CODE_LABEL_NUMBER to -1,
	     otherwise there would be code generation differences
	     in between -g and -g0.  */
	  if (NOTE_P (insn) && NOTE_KIND (insn)
	      == NOTE_INSN_DELETED_DEBUG_LABEL)
	    deleted_debug_label = insn;
	  insn = PREV_INSN (insn);
	}

      /* If we have:
	 label:
	    barrier
	  then this needs to be detected, so skip past the barrier.  */

      if (insn && BARRIER_P (insn))
	insn = PREV_INSN (insn);

      /* Up to now we've only seen notes or barriers.  */
      if (insn)
	{
	  if (LABEL_P (insn)
	      || (NOTE_P (insn)
		  && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL))
	    /* Trailing label.  */
	    fputs ("\tnop\n", file);
	  else if (cfun && ! cfun->is_thunk)
	    {
	      /* See if we have a completely empty function body, skipping
	         the special case of the picbase thunk emitted as asm.  */
	      while (insn && ! INSN_P (insn))
		insn = PREV_INSN (insn);
	      /* If we don't find any insns, we've got an empty function body;
		 I.e. completely empty - without a return or branch.  This is
		 taken as the case where a function body has been removed
		 because it contains an inline __builtin_unreachable().  GCC
		 declares that reaching __builtin_unreachable() means UB so
		 we're not obliged to do anything special; however, we want
		 non-zero-sized function bodies.  To meet this, and help the
		 user out, let's trap the case.  */
	      if (insn == NULL)
		fputs ("\tud2\n", file);
	    }
	}
      else if (deleted_debug_label)
	for (insn = deleted_debug_label; insn; insn = NEXT_INSN (insn))
	  if (NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
	    CODE_LABEL_NUMBER (insn) = -1;
    }
}

/* Implement TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY.  */

void
ix86_print_patchable_function_entry (FILE *file,
				     unsigned HOST_WIDE_INT patch_area_size,
				     bool record_p)
{
  if (cfun->machine->function_label_emitted)
    {
      /* NB: When ix86_print_patchable_function_entry is called after
	 function table has been emitted, we have inserted or queued
	 a pseudo UNSPECV_PATCHABLE_AREA instruction at the proper
	 place.  There is nothing to do here.  */
      return;
    }

  default_print_patchable_function_entry (file, patch_area_size,
					  record_p);
}

/* Output patchable area.  NB: default_print_patchable_function_entry
   isn't available in i386.md.  */

void
ix86_output_patchable_area (unsigned int patch_area_size,
			    bool record_p)
{
  default_print_patchable_function_entry (asm_out_file,
					  patch_area_size,
					  record_p);
}

/* Return a scratch register to use in the split stack prologue.  The
   split stack prologue is used for -fsplit-stack.  It is the first
   instructions in the function, even before the regular prologue.
   The scratch register can be any caller-saved register which is not
   used for parameters or for the static chain.  */

static unsigned int
split_stack_prologue_scratch_regno (void)
{
  if (TARGET_64BIT)
    return R11_REG;
  else
    {
      bool is_fastcall, is_thiscall;
      int regparm;

      is_fastcall = (lookup_attribute ("fastcall",
				       TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
		     != NULL);
      is_thiscall = (lookup_attribute ("thiscall",
				       TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
		     != NULL);
      regparm = ix86_function_regparm (TREE_TYPE (cfun->decl), cfun->decl);

      if (is_fastcall)
	{
	  if (DECL_STATIC_CHAIN (cfun->decl))
	    {
	      sorry ("%<-fsplit-stack%> does not support fastcall with "
		     "nested function");
	      return INVALID_REGNUM;
	    }
	  return AX_REG;
	}
      else if (is_thiscall)
        {
	  if (!DECL_STATIC_CHAIN (cfun->decl))
	    return DX_REG;
	  return AX_REG;
	}
      else if (regparm < 3)
	{
	  if (!DECL_STATIC_CHAIN (cfun->decl))
	    return CX_REG;
	  else
	    {
	      if (regparm >= 2)
		{
		  sorry ("%<-fsplit-stack%> does not support 2 register "
			 "parameters for a nested function");
		  return INVALID_REGNUM;
		}
	      return DX_REG;
	    }
	}
      else
	{
	  /* FIXME: We could make this work by pushing a register
	     around the addition and comparison.  */
	  sorry ("%<-fsplit-stack%> does not support 3 register parameters");
	  return INVALID_REGNUM;
	}
    }
}

/* A SYMBOL_REF for the function which allocates new stackspace for
   -fsplit-stack.  */

static GTY(()) rtx split_stack_fn;

/* A SYMBOL_REF for the more stack function when using the large
   model.  */

static GTY(()) rtx split_stack_fn_large;

/* Return location of the stack guard value in the TLS block.  */

rtx
ix86_split_stack_guard (void)
{
  int offset;
  addr_space_t as = DEFAULT_TLS_SEG_REG;
  rtx r;

  gcc_assert (flag_split_stack);

#ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
  offset = TARGET_THREAD_SPLIT_STACK_OFFSET;
#else
  gcc_unreachable ();
#endif

  r = GEN_INT (offset);
  r = gen_const_mem (Pmode, r);
  set_mem_addr_space (r, as);

  return r;
}

/* Handle -fsplit-stack.  These are the first instructions in the
   function, even before the regular prologue.  */

void
ix86_expand_split_stack_prologue (void)
{
  HOST_WIDE_INT allocate;
  unsigned HOST_WIDE_INT args_size;
  rtx_code_label *label;
  rtx limit, current, allocate_rtx, call_fusage;
  rtx_insn *call_insn;
  rtx scratch_reg = NULL_RTX;
  rtx_code_label *varargs_label = NULL;
  rtx fn;

  gcc_assert (flag_split_stack && reload_completed);

  ix86_finalize_stack_frame_flags ();
  struct ix86_frame &frame = cfun->machine->frame;
  allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET;

  /* This is the label we will branch to if we have enough stack
     space.  We expect the basic block reordering pass to reverse this
     branch if optimizing, so that we branch in the unlikely case.  */
  label = gen_label_rtx ();

  /* We need to compare the stack pointer minus the frame size with
     the stack boundary in the TCB.  The stack boundary always gives
     us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
     can compare directly.  Otherwise we need to do an addition.  */

  limit = ix86_split_stack_guard ();

  if (allocate < SPLIT_STACK_AVAILABLE)
    current = stack_pointer_rtx;
  else
    {
      unsigned int scratch_regno;
      rtx offset;

      /* We need a scratch register to hold the stack pointer minus
	 the required frame size.  Since this is the very start of the
	 function, the scratch register can be any caller-saved
	 register which is not used for parameters.  */
      offset = GEN_INT (- allocate);
      scratch_regno = split_stack_prologue_scratch_regno ();
      if (scratch_regno == INVALID_REGNUM)
	return;
      scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
      if (!TARGET_64BIT || x86_64_immediate_operand (offset, Pmode))
	{
	  /* We don't use gen_add in this case because it will
	     want to split to lea, but when not optimizing the insn
	     will not be split after this point.  */
	  emit_insn (gen_rtx_SET (scratch_reg,
				  gen_rtx_PLUS (Pmode, stack_pointer_rtx,
						offset)));
	}
      else
	{
	  emit_move_insn (scratch_reg, offset);
	  emit_insn (gen_add2_insn (scratch_reg, stack_pointer_rtx));
	}
      current = scratch_reg;
    }

  ix86_expand_branch (GEU, current, limit, label);
  rtx_insn *jump_insn = get_last_insn ();
  JUMP_LABEL (jump_insn) = label;

  /* Mark the jump as very likely to be taken.  */
  add_reg_br_prob_note (jump_insn, profile_probability::very_likely ());

  if (split_stack_fn == NULL_RTX)
    {
      split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack");
      SYMBOL_REF_FLAGS (split_stack_fn) |= SYMBOL_FLAG_LOCAL;
    }
  fn = split_stack_fn;

  /* Get more stack space.  We pass in the desired stack space and the
     size of the arguments to copy to the new stack.  In 32-bit mode
     we push the parameters; __morestack will return on a new stack
     anyhow.  In 64-bit mode we pass the parameters in r10 and
     r11.  */
  allocate_rtx = GEN_INT (allocate);
  args_size = crtl->args.size >= 0 ? (HOST_WIDE_INT) crtl->args.size : 0;
  call_fusage = NULL_RTX;
  rtx pop = NULL_RTX;
  if (TARGET_64BIT)
    {
      rtx reg10, reg11;

      reg10 = gen_rtx_REG (Pmode, R10_REG);
      reg11 = gen_rtx_REG (Pmode, R11_REG);

      /* If this function uses a static chain, it will be in %r10.
	 Preserve it across the call to __morestack.  */
      if (DECL_STATIC_CHAIN (cfun->decl))
	{
	  rtx rax;

	  rax = gen_rtx_REG (word_mode, AX_REG);
	  emit_move_insn (rax, gen_rtx_REG (word_mode, R10_REG));
	  use_reg (&call_fusage, rax);
	}

      if ((ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
          && !TARGET_PECOFF)
	{
	  HOST_WIDE_INT argval;

	  gcc_assert (Pmode == DImode);
	  /* When using the large model we need to load the address
	     into a register, and we've run out of registers.  So we
	     switch to a different calling convention, and we call a
	     different function: __morestack_large.  We pass the
	     argument size in the upper 32 bits of r10 and pass the
	     frame size in the lower 32 bits.  */
	  gcc_assert ((allocate & HOST_WIDE_INT_C (0xffffffff)) == allocate);
	  gcc_assert ((args_size & 0xffffffff) == args_size);

	  if (split_stack_fn_large == NULL_RTX)
	    {
	      split_stack_fn_large
		= gen_rtx_SYMBOL_REF (Pmode, "__morestack_large_model");
	      SYMBOL_REF_FLAGS (split_stack_fn_large) |= SYMBOL_FLAG_LOCAL;
	    }
	  if (ix86_cmodel == CM_LARGE_PIC)
	    {
	      rtx_code_label *label;
	      rtx x;

	      label = gen_label_rtx ();
	      emit_label (label);
	      LABEL_PRESERVE_P (label) = 1;
	      emit_insn (gen_set_rip_rex64 (reg10, label));
	      emit_insn (gen_set_got_offset_rex64 (reg11, label));
	      emit_insn (gen_add2_insn (reg10, reg11));
	      x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, split_stack_fn_large),
				  UNSPEC_GOT);
	      x = gen_rtx_CONST (Pmode, x);
	      emit_move_insn (reg11, x);
	      x = gen_rtx_PLUS (Pmode, reg10, reg11);
	      x = gen_const_mem (Pmode, x);
	      emit_move_insn (reg11, x);
	    }
	  else
	    emit_move_insn (reg11, split_stack_fn_large);

	  fn = reg11;

	  argval = ((args_size << 16) << 16) + allocate;
	  emit_move_insn (reg10, GEN_INT (argval));
	}
      else
	{
	  emit_move_insn (reg10, allocate_rtx);
	  emit_move_insn (reg11, GEN_INT (args_size));
	  use_reg (&call_fusage, reg11);
	}

      use_reg (&call_fusage, reg10);
    }
  else
    {
      rtx_insn *insn = emit_insn (gen_push (GEN_INT (args_size)));
      add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (UNITS_PER_WORD));
      insn = emit_insn (gen_push (allocate_rtx));
      add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (2 * UNITS_PER_WORD));
      pop = GEN_INT (2 * UNITS_PER_WORD);
    }
  call_insn = ix86_expand_call (NULL_RTX, gen_rtx_MEM (QImode, fn),
				GEN_INT (UNITS_PER_WORD), constm1_rtx,
				pop, false);
  add_function_usage_to (call_insn, call_fusage);
  if (!TARGET_64BIT)
    add_reg_note (call_insn, REG_ARGS_SIZE, GEN_INT (0));
  /* Indicate that this function can't jump to non-local gotos.  */
  make_reg_eh_region_note_nothrow_nononlocal (call_insn);

  /* In order to make call/return prediction work right, we now need
     to execute a return instruction.  See
     libgcc/config/i386/morestack.S for the details on how this works.

     For flow purposes gcc must not see this as a return
     instruction--we need control flow to continue at the subsequent
     label.  Therefore, we use an unspec.  */
  gcc_assert (crtl->args.pops_args < 65536);
  rtx_insn *ret_insn
    = emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args)));

  if ((flag_cf_protection & CF_BRANCH))
    {
      /* Insert ENDBR since __morestack will jump back here via indirect
	 call.  */
      rtx cet_eb = gen_nop_endbr ();
      emit_insn_after (cet_eb, ret_insn);
    }

  /* If we are in 64-bit mode and this function uses a static chain,
     we saved %r10 in %rax before calling _morestack.  */
  if (TARGET_64BIT && DECL_STATIC_CHAIN (cfun->decl))
    emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
		    gen_rtx_REG (word_mode, AX_REG));

  /* If this function calls va_start, we need to store a pointer to
     the arguments on the old stack, because they may not have been
     all copied to the new stack.  At this point the old stack can be
     found at the frame pointer value used by __morestack, because
     __morestack has set that up before calling back to us.  Here we
     store that pointer in a scratch register, and in
     ix86_expand_prologue we store the scratch register in a stack
     slot.  */
  if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
    {
      unsigned int scratch_regno;
      rtx frame_reg;
      int words;

      scratch_regno = split_stack_prologue_scratch_regno ();
      scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
      frame_reg = gen_rtx_REG (Pmode, BP_REG);

      /* 64-bit:
	 fp -> old fp value
	       return address within this function
	       return address of caller of this function
	       stack arguments
	 So we add three words to get to the stack arguments.

	 32-bit:
	 fp -> old fp value
	       return address within this function
               first argument to __morestack
               second argument to __morestack
               return address of caller of this function
               stack arguments
         So we add five words to get to the stack arguments.
      */
      words = TARGET_64BIT ? 3 : 5;
      emit_insn (gen_rtx_SET (scratch_reg,
			      plus_constant (Pmode, frame_reg,
					     words * UNITS_PER_WORD)));

      varargs_label = gen_label_rtx ();
      emit_jump_insn (gen_jump (varargs_label));
      JUMP_LABEL (get_last_insn ()) = varargs_label;

      emit_barrier ();
    }

  emit_label (label);
  LABEL_NUSES (label) = 1;

  /* If this function calls va_start, we now have to set the scratch
     register for the case where we do not call __morestack.  In this
     case we need to set it based on the stack pointer.  */
  if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
    {
      emit_insn (gen_rtx_SET (scratch_reg,
			      plus_constant (Pmode, stack_pointer_rtx,
					     UNITS_PER_WORD)));

      emit_label (varargs_label);
      LABEL_NUSES (varargs_label) = 1;
    }
}

/* We may have to tell the dataflow pass that the split stack prologue
   is initializing a scratch register.  */

static void
ix86_live_on_entry (bitmap regs)
{
  if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
    {
      gcc_assert (flag_split_stack);
      bitmap_set_bit (regs, split_stack_prologue_scratch_regno ());
    }
}

/* Extract the parts of an RTL expression that is a valid memory address
   for an instruction.  Return false if the structure of the address is
   grossly off.  */

bool
ix86_decompose_address (rtx addr, struct ix86_address *out)
{
  rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
  rtx base_reg, index_reg;
  HOST_WIDE_INT scale = 1;
  rtx scale_rtx = NULL_RTX;
  rtx tmp;
  addr_space_t seg = ADDR_SPACE_GENERIC;

  /* Allow zero-extended SImode addresses,
     they will be emitted with addr32 prefix.  */
  if (TARGET_64BIT && GET_MODE (addr) == DImode)
    {
      if (GET_CODE (addr) == ZERO_EXTEND
	  && GET_MODE (XEXP (addr, 0)) == SImode)
	{
	  addr = XEXP (addr, 0);
	  if (CONST_INT_P (addr))
	    return false;
	}	      
      else if (GET_CODE (addr) == AND
	       && const_32bit_mask (XEXP (addr, 1), DImode))
	{
	  addr = lowpart_subreg (SImode, XEXP (addr, 0), DImode);
	  if (addr == NULL_RTX)
	    return false;

	  if (CONST_INT_P (addr))
	    return false;
	}
      else if (GET_CODE (addr) == AND)
	{
	  /* For ASHIFT inside AND, combine will not generate
	     canonical zero-extend. Merge mask for AND and shift_count
	     to check if it is canonical zero-extend.  */
	  tmp = XEXP (addr, 0);
	  rtx mask = XEXP (addr, 1);
	  if (tmp && GET_CODE(tmp) == ASHIFT)
	    {
	      rtx shift_val = XEXP (tmp, 1);
	      if (CONST_INT_P (mask) && CONST_INT_P (shift_val)
		  && (((unsigned HOST_WIDE_INT) INTVAL(mask)
		      | ((HOST_WIDE_INT_1U << INTVAL(shift_val)) - 1))
		      == 0xffffffff))
		{
		  addr = lowpart_subreg (SImode, XEXP (addr, 0),
					 DImode);
		}
	    }

	}
    }

  /* Allow SImode subregs of DImode addresses,
     they will be emitted with addr32 prefix.  */
  if (TARGET_64BIT && GET_MODE (addr) == SImode)
    {
      if (SUBREG_P (addr)
	  && GET_MODE (SUBREG_REG (addr)) == DImode)
	{
	  addr = SUBREG_REG (addr);
	  if (CONST_INT_P (addr))
	    return false;
	}
    }

  if (REG_P (addr))
    base = addr;
  else if (SUBREG_P (addr))
    {
      if (REG_P (SUBREG_REG (addr)))
	base = addr;
      else
	return false;
    }
  else if (GET_CODE (addr) == PLUS)
    {
      rtx addends[4], op;
      int n = 0, i;

      op = addr;
      do
	{
	  if (n >= 4)
	    return false;
	  addends[n++] = XEXP (op, 1);
	  op = XEXP (op, 0);
	}
      while (GET_CODE (op) == PLUS);
      if (n >= 4)
	return false;
      addends[n] = op;

      for (i = n; i >= 0; --i)
	{
	  op = addends[i];
	  switch (GET_CODE (op))
	    {
	    case MULT:
	      if (index)
		return false;
	      index = XEXP (op, 0);
	      scale_rtx = XEXP (op, 1);
	      break;

	    case ASHIFT:
	      if (index)
		return false;
	      index = XEXP (op, 0);
	      tmp = XEXP (op, 1);
	      if (!CONST_INT_P (tmp))
		return false;
	      scale = INTVAL (tmp);
	      if ((unsigned HOST_WIDE_INT) scale > 3)
		return false;
	      scale = 1 << scale;
	      break;

	    case ZERO_EXTEND:
	      op = XEXP (op, 0);
	      if (GET_CODE (op) != UNSPEC)
		return false;
	      /* FALLTHRU */

	    case UNSPEC:
	      if (XINT (op, 1) == UNSPEC_TP
	          && TARGET_TLS_DIRECT_SEG_REFS
	          && seg == ADDR_SPACE_GENERIC)
		seg = DEFAULT_TLS_SEG_REG;
	      else
		return false;
	      break;

	    case SUBREG:
	      if (!REG_P (SUBREG_REG (op)))
		return false;
	      /* FALLTHRU */

	    case REG:
	      if (!base)
		base = op;
	      else if (!index)
		index = op;
	      else
		return false;
	      break;

	    case CONST:
	    case CONST_INT:
	    case SYMBOL_REF:
	    case LABEL_REF:
	      if (disp)
		return false;
	      disp = op;
	      break;

	    default:
	      return false;
	    }
	}
    }
  else if (GET_CODE (addr) == MULT)
    {
      index = XEXP (addr, 0);		/* index*scale */
      scale_rtx = XEXP (addr, 1);
    }
  else if (GET_CODE (addr) == ASHIFT)
    {
      /* We're called for lea too, which implements ashift on occasion.  */
      index = XEXP (addr, 0);
      tmp = XEXP (addr, 1);
      if (!CONST_INT_P (tmp))
	return false;
      scale = INTVAL (tmp);
      if ((unsigned HOST_WIDE_INT) scale > 3)
	return false;
      scale = 1 << scale;
    }
  else
    disp = addr;			/* displacement */

  if (index)
    {
      if (REG_P (index))
	;
      else if (SUBREG_P (index)
	       && REG_P (SUBREG_REG (index)))
	;
      else
	return false;
    }

  /* Extract the integral value of scale.  */
  if (scale_rtx)
    {
      if (!CONST_INT_P (scale_rtx))
	return false;
      scale = INTVAL (scale_rtx);
    }

  base_reg = base && SUBREG_P (base) ? SUBREG_REG (base) : base;
  index_reg = index && SUBREG_P (index) ? SUBREG_REG (index) : index;

  /* Avoid useless 0 displacement.  */
  if (disp == const0_rtx && (base || index))
    disp = NULL_RTX;

  /* Allow arg pointer and stack pointer as index if there is not scaling.  */
  if (base_reg && index_reg && scale == 1
      && (REGNO (index_reg) == ARG_POINTER_REGNUM
	  || REGNO (index_reg) == FRAME_POINTER_REGNUM
	  || REGNO (index_reg) == SP_REG))
    {
      std::swap (base, index);
      std::swap (base_reg, index_reg);
    }

  /* Special case: %ebp cannot be encoded as a base without a displacement.
     Similarly %r13.  */
  if (!disp && base_reg
      && (REGNO (base_reg) == ARG_POINTER_REGNUM
	  || REGNO (base_reg) == FRAME_POINTER_REGNUM
	  || REGNO (base_reg) == BP_REG
	  || REGNO (base_reg) == R13_REG))
    disp = const0_rtx;

  /* Special case: on K6, [%esi] makes the instruction vector decoded.
     Avoid this by transforming to [%esi+0].
     Reload calls address legitimization without cfun defined, so we need
     to test cfun for being non-NULL. */
  if (TARGET_CPU_P (K6) && cfun && optimize_function_for_speed_p (cfun)
      && base_reg && !index_reg && !disp
      && REGNO (base_reg) == SI_REG)
    disp = const0_rtx;

  /* Special case: encode reg+reg instead of reg*2.  */
  if (!base && index && scale == 2)
    base = index, base_reg = index_reg, scale = 1;

  /* Special case: scaling cannot be encoded without base or displacement.  */
  if (!base && !disp && index && scale != 1)
    disp = const0_rtx;

  out->base = base;
  out->index = index;
  out->disp = disp;
  out->scale = scale;
  out->seg = seg;

  return true;
}

/* Return cost of the memory address x.
   For i386, it is better to use a complex address than let gcc copy
   the address into a reg and make a new pseudo.  But not if the address
   requires to two regs - that would mean more pseudos with longer
   lifetimes.  */
static int
ix86_address_cost (rtx x, machine_mode, addr_space_t, bool)
{
  struct ix86_address parts;
  int cost = 1;
  int ok = ix86_decompose_address (x, &parts);

  gcc_assert (ok);

  if (parts.base && SUBREG_P (parts.base))
    parts.base = SUBREG_REG (parts.base);
  if (parts.index && SUBREG_P (parts.index))
    parts.index = SUBREG_REG (parts.index);

  /* Attempt to minimize number of registers in the address by increasing
     address cost for each used register.  We don't increase address cost
     for "pic_offset_table_rtx".  When a memopt with "pic_offset_table_rtx"
     is not invariant itself it most likely means that base or index is not
     invariant.  Therefore only "pic_offset_table_rtx" could be hoisted out,
     which is not profitable for x86.  */
  if (parts.base
      && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER)
      && (current_pass->type == GIMPLE_PASS
	  || !pic_offset_table_rtx
	  || !REG_P (parts.base)
	  || REGNO (pic_offset_table_rtx) != REGNO (parts.base)))
    cost++;

  if (parts.index
      && (!REG_P (parts.index) || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)
      && (current_pass->type == GIMPLE_PASS
	  || !pic_offset_table_rtx
	  || !REG_P (parts.index)
	  || REGNO (pic_offset_table_rtx) != REGNO (parts.index)))
    cost++;

  /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
     since it's predecode logic can't detect the length of instructions
     and it degenerates to vector decoded.  Increase cost of such
     addresses here.  The penalty is minimally 2 cycles.  It may be worthwhile
     to split such addresses or even refuse such addresses at all.

     Following addressing modes are affected:
      [base+scale*index]
      [scale*index+disp]
      [base+index]

     The first and last case  may be avoidable by explicitly coding the zero in
     memory address, but I don't have AMD-K6 machine handy to check this
     theory.  */

  if (TARGET_CPU_P (K6)
      && ((!parts.disp && parts.base && parts.index && parts.scale != 1)
	  || (parts.disp && !parts.base && parts.index && parts.scale != 1)
	  || (!parts.disp && parts.base && parts.index && parts.scale == 1)))
    cost += 10;

  return cost;
}

/* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
   this is used for to form addresses to local data when -fPIC is in
   use.  */

static bool
darwin_local_data_pic (rtx disp)
{
  return (GET_CODE (disp) == UNSPEC
	  && XINT (disp, 1) == UNSPEC_MACHOPIC_OFFSET);
}

/* True if the function symbol operand X should be loaded from GOT.
   If CALL_P is true, X is a call operand.

   NB: -mno-direct-extern-access doesn't force load from GOT for
   call.

   NB: In 32-bit mode, only non-PIC is allowed in inline assembly
   statements, since a PIC register could not be available at the
   call site.  */

bool
ix86_force_load_from_GOT_p (rtx x, bool call_p)
{
  return ((TARGET_64BIT || (!flag_pic && HAVE_AS_IX86_GOT32X))
	  && !TARGET_PECOFF && !TARGET_MACHO
	  && (!flag_pic || this_is_asm_operands)
	  && ix86_cmodel != CM_LARGE
	  && ix86_cmodel != CM_LARGE_PIC
	  && GET_CODE (x) == SYMBOL_REF
	  && ((!call_p
	       && (!ix86_direct_extern_access
		   || (SYMBOL_REF_DECL (x)
		       && lookup_attribute ("nodirect_extern_access",
					    DECL_ATTRIBUTES (SYMBOL_REF_DECL (x))))))
	      || (SYMBOL_REF_FUNCTION_P (x)
		  && (!flag_plt
		      || (SYMBOL_REF_DECL (x)
			  && lookup_attribute ("noplt",
					       DECL_ATTRIBUTES (SYMBOL_REF_DECL (x)))))))
	  && !SYMBOL_REF_LOCAL_P (x));
}

/* Determine if a given RTX is a valid constant.  We already know this
   satisfies CONSTANT_P.  */

static bool
ix86_legitimate_constant_p (machine_mode mode, rtx x)
{
  switch (GET_CODE (x))
    {
    case CONST:
      x = XEXP (x, 0);

      if (GET_CODE (x) == PLUS)
	{
	  if (!CONST_INT_P (XEXP (x, 1)))
	    return false;
	  x = XEXP (x, 0);
	}

      if (TARGET_MACHO && darwin_local_data_pic (x))
	return true;

      /* Only some unspecs are valid as "constants".  */
      if (GET_CODE (x) == UNSPEC)
	switch (XINT (x, 1))
	  {
	  case UNSPEC_GOT:
	  case UNSPEC_GOTOFF:
	  case UNSPEC_PLTOFF:
	    return TARGET_64BIT;
	  case UNSPEC_TPOFF:
	  case UNSPEC_NTPOFF:
	    x = XVECEXP (x, 0, 0);
	    return (GET_CODE (x) == SYMBOL_REF
		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
	  case UNSPEC_DTPOFF:
	    x = XVECEXP (x, 0, 0);
	    return (GET_CODE (x) == SYMBOL_REF
		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
	  default:
	    return false;
	  }

      /* We must have drilled down to a symbol.  */
      if (GET_CODE (x) == LABEL_REF)
	return true;
      if (GET_CODE (x) != SYMBOL_REF)
	return false;
      /* FALLTHRU */

    case SYMBOL_REF:
      /* TLS symbols are never valid.  */
      if (SYMBOL_REF_TLS_MODEL (x))
	return false;

      /* DLLIMPORT symbols are never valid.  */
      if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
	  && SYMBOL_REF_DLLIMPORT_P (x))
	return false;

#if TARGET_MACHO
      /* mdynamic-no-pic */
      if (MACHO_DYNAMIC_NO_PIC_P)
	return machopic_symbol_defined_p (x);
#endif

      /* External function address should be loaded
	 via the GOT slot to avoid PLT.  */
      if (ix86_force_load_from_GOT_p (x))
	return false;

      break;

    CASE_CONST_SCALAR_INT:
      if (ix86_endbr_immediate_operand (x, VOIDmode))
	return false;

      switch (mode)
	{
	case E_TImode:
	  if (TARGET_64BIT)
	    return true;
	  /* FALLTHRU */
	case E_OImode:
	case E_XImode:
	  if (!standard_sse_constant_p (x, mode)
	      && GET_MODE_SIZE (TARGET_AVX512F
				? XImode
				: (TARGET_AVX
				   ? OImode
				   : (TARGET_SSE2
				      ? TImode : DImode))) < GET_MODE_SIZE (mode))
	    return false;
	default:
	  break;
	}
      break;

    case CONST_VECTOR:
      if (!standard_sse_constant_p (x, mode))
	return false;
      break;

    case CONST_DOUBLE:
      if (mode == E_BFmode)
	return false;

    default:
      break;
    }

  /* Otherwise we handle everything else in the move patterns.  */
  return true;
}

/* Determine if it's legal to put X into the constant pool.  This
   is not possible for the address of thread-local symbols, which
   is checked above.  */

static bool
ix86_cannot_force_const_mem (machine_mode mode, rtx x)
{
  /* We can put any immediate constant in memory.  */
  switch (GET_CODE (x))
    {
    CASE_CONST_ANY:
      return false;

    default:
      break;
    }

  return !ix86_legitimate_constant_p (mode, x);
}

/*  Nonzero if the symbol is marked as dllimport, or as stub-variable,
    otherwise zero.  */

static bool
is_imported_p (rtx x)
{
  if (!TARGET_DLLIMPORT_DECL_ATTRIBUTES
      || GET_CODE (x) != SYMBOL_REF)
    return false;

  return SYMBOL_REF_DLLIMPORT_P (x) || SYMBOL_REF_STUBVAR_P (x);
}


/* Nonzero if the constant value X is a legitimate general operand
   when generating PIC code.  It is given that flag_pic is on and
   that X satisfies CONSTANT_P.  */

bool
legitimate_pic_operand_p (rtx x)
{
  rtx inner;

  switch (GET_CODE (x))
    {
    case CONST:
      inner = XEXP (x, 0);
      if (GET_CODE (inner) == PLUS
	  && CONST_INT_P (XEXP (inner, 1)))
	inner = XEXP (inner, 0);

      /* Only some unspecs are valid as "constants".  */
      if (GET_CODE (inner) == UNSPEC)
	switch (XINT (inner, 1))
	  {
	  case UNSPEC_GOT:
	  case UNSPEC_GOTOFF:
	  case UNSPEC_PLTOFF:
	    return TARGET_64BIT;
	  case UNSPEC_TPOFF:
	    x = XVECEXP (inner, 0, 0);
	    return (GET_CODE (x) == SYMBOL_REF
		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
	  case UNSPEC_MACHOPIC_OFFSET:
	    return legitimate_pic_address_disp_p (x);
	  default:
	    return false;
	  }
      /* FALLTHRU */

    case SYMBOL_REF:
    case LABEL_REF:
      return legitimate_pic_address_disp_p (x);

    default:
      return true;
    }
}

/* Determine if a given CONST RTX is a valid memory displacement
   in PIC mode.  */

bool
legitimate_pic_address_disp_p (rtx disp)
{
  bool saw_plus;

  /* In 64bit mode we can allow direct addresses of symbols and labels
     when they are not dynamic symbols.  */
  if (TARGET_64BIT)
    {
      rtx op0 = disp, op1;

      switch (GET_CODE (disp))
	{
	case LABEL_REF:
	  return true;

	case CONST:
	  if (GET_CODE (XEXP (disp, 0)) != PLUS)
	    break;
	  op0 = XEXP (XEXP (disp, 0), 0);
	  op1 = XEXP (XEXP (disp, 0), 1);
	  if (!CONST_INT_P (op1))
	    break;
	  if (GET_CODE (op0) == UNSPEC
	      && (XINT (op0, 1) == UNSPEC_DTPOFF
		  || XINT (op0, 1) == UNSPEC_NTPOFF)
	      && trunc_int_for_mode (INTVAL (op1), SImode) == INTVAL (op1))
	    return true;
	  if (INTVAL (op1) >= 16*1024*1024
	      || INTVAL (op1) < -16*1024*1024)
	    break;
	  if (GET_CODE (op0) == LABEL_REF)
	    return true;
	  if (GET_CODE (op0) == CONST
	      && GET_CODE (XEXP (op0, 0)) == UNSPEC
	      && XINT (XEXP (op0, 0), 1) == UNSPEC_PCREL)
	    return true;
	  if (GET_CODE (op0) == UNSPEC
	      && XINT (op0, 1) == UNSPEC_PCREL)
	    return true;
	  if (GET_CODE (op0) != SYMBOL_REF)
	    break;
	  /* FALLTHRU */

	case SYMBOL_REF:
	  /* TLS references should always be enclosed in UNSPEC.
	     The dllimported symbol needs always to be resolved.  */
	  if (SYMBOL_REF_TLS_MODEL (op0)
	      || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && SYMBOL_REF_DLLIMPORT_P (op0)))
	    return false;

	  if (TARGET_PECOFF)
	    {
	      if (is_imported_p (op0))
		return true;

	      if (SYMBOL_REF_FAR_ADDR_P (op0) || !SYMBOL_REF_LOCAL_P (op0))
		break;

	      /* Non-external-weak function symbols need to be resolved only
		 for the large model.  Non-external symbols don't need to be
		 resolved for large and medium models.  For the small model,
		 we don't need to resolve anything here.  */
	      if ((ix86_cmodel != CM_LARGE_PIC
		   && SYMBOL_REF_FUNCTION_P (op0)
		   && !(SYMBOL_REF_EXTERNAL_P (op0) && SYMBOL_REF_WEAK (op0)))
		  || !SYMBOL_REF_EXTERNAL_P (op0)
		  || ix86_cmodel == CM_SMALL_PIC)
		return true;
	    }
	  else if (!SYMBOL_REF_FAR_ADDR_P (op0)
		   && (SYMBOL_REF_LOCAL_P (op0)
		       || ((ix86_direct_extern_access
			    && !(SYMBOL_REF_DECL (op0)
				 && lookup_attribute ("nodirect_extern_access",
						      DECL_ATTRIBUTES (SYMBOL_REF_DECL (op0)))))
			   && HAVE_LD_PIE_COPYRELOC
			   && flag_pie
			   && !SYMBOL_REF_WEAK (op0)
			   && !SYMBOL_REF_FUNCTION_P (op0)))
		   && ix86_cmodel != CM_LARGE_PIC)
	    return true;
	  break;

	default:
	  break;
	}
    }
  if (GET_CODE (disp) != CONST)
    return false;
  disp = XEXP (disp, 0);

  if (TARGET_64BIT)
    {
      /* We are unsafe to allow PLUS expressions.  This limit allowed distance
         of GOT tables.  We should not need these anyway.  */
      if (GET_CODE (disp) != UNSPEC
	  || (XINT (disp, 1) != UNSPEC_GOTPCREL
	      && XINT (disp, 1) != UNSPEC_GOTOFF
	      && XINT (disp, 1) != UNSPEC_PCREL
	      && XINT (disp, 1) != UNSPEC_PLTOFF))
	return false;

      if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF
	  && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF)
	return false;
      return true;
    }

  saw_plus = false;
  if (GET_CODE (disp) == PLUS)
    {
      if (!CONST_INT_P (XEXP (disp, 1)))
	return false;
      disp = XEXP (disp, 0);
      saw_plus = true;
    }

  if (TARGET_MACHO && darwin_local_data_pic (disp))
    return true;

  if (GET_CODE (disp) != UNSPEC)
    return false;

  switch (XINT (disp, 1))
    {
    case UNSPEC_GOT:
      if (saw_plus)
	return false;
      /* We need to check for both symbols and labels because VxWorks loads
	 text labels with @GOT rather than @GOTOFF.  See gotoff_operand for
	 details.  */
      return (GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
	      || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF);
    case UNSPEC_GOTOFF:
      /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
	 While ABI specify also 32bit relocation but we don't produce it in
	 small PIC model at all.  */
      if ((GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
	   || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF)
	  && !TARGET_64BIT)
        return !TARGET_PECOFF && gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
      return false;
    case UNSPEC_GOTTPOFF:
    case UNSPEC_GOTNTPOFF:
    case UNSPEC_INDNTPOFF:
      if (saw_plus)
	return false;
      disp = XVECEXP (disp, 0, 0);
      return (GET_CODE (disp) == SYMBOL_REF
	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
    case UNSPEC_NTPOFF:
      disp = XVECEXP (disp, 0, 0);
      return (GET_CODE (disp) == SYMBOL_REF
	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
    case UNSPEC_DTPOFF:
      disp = XVECEXP (disp, 0, 0);
      return (GET_CODE (disp) == SYMBOL_REF
	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
    }

  return false;
}

/* Determine if op is suitable RTX for an address register.
   Return naked register if a register or a register subreg is
   found, otherwise return NULL_RTX.  */

static rtx
ix86_validate_address_register (rtx op)
{
  machine_mode mode = GET_MODE (op);

  /* Only SImode or DImode registers can form the address.  */
  if (mode != SImode && mode != DImode)
    return NULL_RTX;

  if (REG_P (op))
    return op;
  else if (SUBREG_P (op))
    {
      rtx reg = SUBREG_REG (op);

      if (!REG_P (reg))
	return NULL_RTX;

      mode = GET_MODE (reg);

      /* Don't allow SUBREGs that span more than a word.  It can
	 lead to spill failures when the register is one word out
	 of a two word structure.  */
      if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	return NULL_RTX;

      /* Allow only SUBREGs of non-eliminable hard registers.  */
      if (register_no_elim_operand (reg, mode))
	return reg;
    }

  /* Op is not a register.  */
  return NULL_RTX;
}

/* Recognizes RTL expressions that are valid memory addresses for an
   instruction.  The MODE argument is the machine mode for the MEM
   expression that wants to use this address.

   It only recognizes address in canonical form.  LEGITIMIZE_ADDRESS should
   convert common non-canonical forms to canonical form so that they will
   be recognized.  */

static bool
ix86_legitimate_address_p (machine_mode, rtx addr, bool strict,
			   code_helper = ERROR_MARK)
{
  struct ix86_address parts;
  rtx base, index, disp;
  HOST_WIDE_INT scale;
  addr_space_t seg;

  if (ix86_decompose_address (addr, &parts) == 0)
    /* Decomposition failed.  */
    return false;

  base = parts.base;
  index = parts.index;
  disp = parts.disp;
  scale = parts.scale;
  seg = parts.seg;

  /* Validate base register.  */
  if (base)
    {
      rtx reg = ix86_validate_address_register (base);

      if (reg == NULL_RTX)
	return false;

      unsigned int regno = REGNO (reg);
      if ((strict && !REGNO_OK_FOR_BASE_P (regno))
	  || (!strict && !REGNO_OK_FOR_BASE_NONSTRICT_P (regno)))
	/* Base is not valid.  */
	return false;
    }

  /* Validate index register.  */
  if (index)
    {
      rtx reg = ix86_validate_address_register (index);

      if (reg == NULL_RTX)
	return false;

      unsigned int regno = REGNO (reg);
      if ((strict && !REGNO_OK_FOR_INDEX_P (regno))
	  || (!strict && !REGNO_OK_FOR_INDEX_NONSTRICT_P (regno)))
	/* Index is not valid.  */
	return false;
    }

  /* Index and base should have the same mode.  */
  if (base && index
      && GET_MODE (base) != GET_MODE (index))
    return false;

  /* Address override works only on the (%reg) part of %fs:(%reg).  */
  if (seg != ADDR_SPACE_GENERIC
      && ((base && GET_MODE (base) != word_mode)
	  || (index && GET_MODE (index) != word_mode)))
    return false;

  /* Validate scale factor.  */
  if (scale != 1)
    {
      if (!index)
	/* Scale without index.  */
	return false;

      if (scale != 2 && scale != 4 && scale != 8)
	/* Scale is not a valid multiplier.  */
	return false;
    }

  /* Validate displacement.  */
  if (disp)
    {
      if (ix86_endbr_immediate_operand (disp, VOIDmode))
	return false;

      if (GET_CODE (disp) == CONST
	  && GET_CODE (XEXP (disp, 0)) == UNSPEC
	  && XINT (XEXP (disp, 0), 1) != UNSPEC_MACHOPIC_OFFSET)
	switch (XINT (XEXP (disp, 0), 1))
	  {
	  /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit
	     when used.  While ABI specify also 32bit relocations, we
	     don't produce them at all and use IP relative instead.
	     Allow GOT in 32bit mode for both PIC and non-PIC if symbol
	     should be loaded via GOT.  */
	  case UNSPEC_GOT:
	    if (!TARGET_64BIT
		&& ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
	      goto is_legitimate_pic;
	    /* FALLTHRU */
	  case UNSPEC_GOTOFF:
	    gcc_assert (flag_pic);
	    if (!TARGET_64BIT)
	      goto is_legitimate_pic;

	    /* 64bit address unspec.  */
	    return false;

	  case UNSPEC_GOTPCREL:
	    if (ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
	      goto is_legitimate_pic;
	    /* FALLTHRU */
	  case UNSPEC_PCREL:
	    gcc_assert (flag_pic);
	    goto is_legitimate_pic;

	  case UNSPEC_GOTTPOFF:
	  case UNSPEC_GOTNTPOFF:
	  case UNSPEC_INDNTPOFF:
	  case UNSPEC_NTPOFF:
	  case UNSPEC_DTPOFF:
	    break;

	  default:
	    /* Invalid address unspec.  */
	    return false;
	  }

      else if (SYMBOLIC_CONST (disp)
	       && (flag_pic
#if TARGET_MACHO
		   || (MACHOPIC_INDIRECT
		       && !machopic_operand_p (disp))
#endif
		  ))
	{

	is_legitimate_pic:
	  if (TARGET_64BIT && (index || base))
	    {
	      /* foo@dtpoff(%rX) is ok.  */
	      if (GET_CODE (disp) != CONST
		  || GET_CODE (XEXP (disp, 0)) != PLUS
		  || GET_CODE (XEXP (XEXP (disp, 0), 0)) != UNSPEC
		  || !CONST_INT_P (XEXP (XEXP (disp, 0), 1))
		  || (XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_DTPOFF
		      && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_NTPOFF))
		/* Non-constant pic memory reference.  */
		return false;
	    }
	  else if ((!TARGET_MACHO || flag_pic)
		    && ! legitimate_pic_address_disp_p (disp))
	    /* Displacement is an invalid pic construct.  */
	    return false;
#if TARGET_MACHO
	  else if (MACHO_DYNAMIC_NO_PIC_P
		   && !ix86_legitimate_constant_p (Pmode, disp))
	    /* displacment must be referenced via non_lazy_pointer */
	    return false;
#endif

          /* This code used to verify that a symbolic pic displacement
	     includes the pic_offset_table_rtx register.

	     While this is good idea, unfortunately these constructs may
	     be created by "adds using lea" optimization for incorrect
	     code like:

	     int a;
	     int foo(int i)
	       {
	         return *(&a+i);
	       }

	     This code is nonsensical, but results in addressing
	     GOT table with pic_offset_table_rtx base.  We can't
	     just refuse it easily, since it gets matched by
	     "addsi3" pattern, that later gets split to lea in the
	     case output register differs from input.  While this
	     can be handled by separate addsi pattern for this case
	     that never results in lea, this seems to be easier and
	     correct fix for crash to disable this test.  */
	}
      else if (GET_CODE (disp) != LABEL_REF
	       && !CONST_INT_P (disp)
	       && (GET_CODE (disp) != CONST
		   || !ix86_legitimate_constant_p (Pmode, disp))
	       && (GET_CODE (disp) != SYMBOL_REF
		   || !ix86_legitimate_constant_p (Pmode, disp)))
	/* Displacement is not constant.  */
	return false;
      else if (TARGET_64BIT
	       && !x86_64_immediate_operand (disp, VOIDmode))
	/* Displacement is out of range.  */
	return false;
      /* In x32 mode, constant addresses are sign extended to 64bit, so
	 we have to prevent addresses from 0x80000000 to 0xffffffff.  */
      else if (TARGET_X32 && !(index || base)
	       && CONST_INT_P (disp)
	       && val_signbit_known_set_p (SImode, INTVAL (disp)))
	return false;
    }

  /* Everything looks valid.  */
  return true;
}

/* Determine if a given RTX is a valid constant address.  */

bool
constant_address_p (rtx x)
{
  return CONSTANT_P (x) && ix86_legitimate_address_p (Pmode, x, 1);
}

/* Return a unique alias set for the GOT.  */

alias_set_type
ix86_GOT_alias_set (void)
{
  static alias_set_type set = -1;
  if (set == -1)
    set = new_alias_set ();
  return set;
}

/* Return a legitimate reference for ORIG (an address) using the
   register REG.  If REG is 0, a new pseudo is generated.

   There are two types of references that must be handled:

   1. Global data references must load the address from the GOT, via
      the PIC reg.  An insn is emitted to do this load, and the reg is
      returned.

   2. Static data references, constant pool addresses, and code labels
      compute the address as an offset from the GOT, whose base is in
      the PIC reg.  Static data objects have SYMBOL_FLAG_LOCAL set to
      differentiate them from global data objects.  The returned
      address is the PIC reg + an unspec constant.

   TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
   reg also appears in the address.  */

rtx
legitimize_pic_address (rtx orig, rtx reg)
{
  rtx addr = orig;
  rtx new_rtx = orig;

#if TARGET_MACHO
  if (TARGET_MACHO && !TARGET_64BIT)
    {
      if (reg == 0)
	reg = gen_reg_rtx (Pmode);
      /* Use the generic Mach-O PIC machinery.  */
      return machopic_legitimize_pic_address (orig, GET_MODE (orig), reg);
    }
#endif

  if (TARGET_64BIT && TARGET_DLLIMPORT_DECL_ATTRIBUTES)
    {
      rtx tmp = legitimize_pe_coff_symbol (addr, true);
      if (tmp)
        return tmp;
    }

  if (TARGET_64BIT && legitimate_pic_address_disp_p (addr))
    new_rtx = addr;
  else if ((!TARGET_64BIT
	    || /* TARGET_64BIT && */ ix86_cmodel != CM_SMALL_PIC)
	   && !TARGET_PECOFF
	   && gotoff_operand (addr, Pmode))
    {
      /* This symbol may be referenced via a displacement
	 from the PIC base address (@GOTOFF).  */
      if (GET_CODE (addr) == CONST)
	addr = XEXP (addr, 0);

      if (GET_CODE (addr) == PLUS)
	  {
            new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
				      UNSPEC_GOTOFF);
	    new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
	  }
	else
          new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);

      new_rtx = gen_rtx_CONST (Pmode, new_rtx);

      if (TARGET_64BIT)
	new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);

      if (reg != 0)
	{
 	  gcc_assert (REG_P (reg));
	  new_rtx = expand_simple_binop (Pmode, PLUS, pic_offset_table_rtx,
					 new_rtx, reg, 1, OPTAB_DIRECT);
 	}
      else
	new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
    }
  else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
	   /* We can't always use @GOTOFF for text labels
	      on VxWorks, see gotoff_operand.  */
	   || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
    {
      rtx tmp = legitimize_pe_coff_symbol (addr, true);
      if (tmp)
        return tmp;

      /* For x64 PE-COFF there is no GOT table,
	 so we use address directly.  */
      if (TARGET_64BIT && TARGET_PECOFF)
	{
	  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_PCREL);
	  new_rtx = gen_rtx_CONST (Pmode, new_rtx);
	}
      else if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC)
	{
	  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
				    UNSPEC_GOTPCREL);
	  new_rtx = gen_rtx_CONST (Pmode, new_rtx);
	  new_rtx = gen_const_mem (Pmode, new_rtx);
	  set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
	}
      else
	{
	  /* This symbol must be referenced via a load
	     from the Global Offset Table (@GOT).  */
	  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
	  new_rtx = gen_rtx_CONST (Pmode, new_rtx);

	  if (TARGET_64BIT)
	    new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);

	  if (reg != 0)
	    {
	      gcc_assert (REG_P (reg));
	      new_rtx = expand_simple_binop (Pmode, PLUS, pic_offset_table_rtx,
					     new_rtx, reg, 1, OPTAB_DIRECT);
	    }
	  else
	    new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);

	  new_rtx = gen_const_mem (Pmode, new_rtx);
	  set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
	}

      new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
    }
  else
    {
      if (CONST_INT_P (addr)
	  && !x86_64_immediate_operand (addr, VOIDmode))
	new_rtx = copy_to_suggested_reg (addr, reg, Pmode);
      else if (GET_CODE (addr) == CONST)
	{
	  addr = XEXP (addr, 0);

	  /* We must match stuff we generate before.  Assume the only
	     unspecs that can get here are ours.  Not that we could do
	     anything with them anyway....  */
	  if (GET_CODE (addr) == UNSPEC
	      || (GET_CODE (addr) == PLUS
		  && GET_CODE (XEXP (addr, 0)) == UNSPEC))
	    return orig;
	  gcc_assert (GET_CODE (addr) == PLUS);
	}

      if (GET_CODE (addr) == PLUS)
	{
	  rtx op0 = XEXP (addr, 0), op1 = XEXP (addr, 1);

	  /* Check first to see if this is a constant
	     offset from a @GOTOFF symbol reference.  */
	  if (!TARGET_PECOFF
	      && gotoff_operand (op0, Pmode)
	      && CONST_INT_P (op1))
	    {
	      if (!TARGET_64BIT)
		{
		  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op0),
					    UNSPEC_GOTOFF);
		  new_rtx = gen_rtx_PLUS (Pmode, new_rtx, op1);
		  new_rtx = gen_rtx_CONST (Pmode, new_rtx);

		  if (reg != 0)
		    {
		      gcc_assert (REG_P (reg));
		      new_rtx = expand_simple_binop (Pmode, PLUS,
						     pic_offset_table_rtx,
						     new_rtx, reg, 1,
						     OPTAB_DIRECT);
		    }
		  else
		    new_rtx
		      = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
		}
	      else
		{
		  if (INTVAL (op1) < -16*1024*1024
		      || INTVAL (op1) >= 16*1024*1024)
		    {
		      if (!x86_64_immediate_operand (op1, Pmode))
			op1 = force_reg (Pmode, op1);

		      new_rtx
			= gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), op1);
		    }
		}
	    }
	  else
	    {
	      rtx base = legitimize_pic_address (op0, reg);
	      machine_mode mode = GET_MODE (base);
	      new_rtx
	        = legitimize_pic_address (op1, base == reg ? NULL_RTX : reg);

	      if (CONST_INT_P (new_rtx))
		{
		  if (INTVAL (new_rtx) < -16*1024*1024
		      || INTVAL (new_rtx) >= 16*1024*1024)
		    {
		      if (!x86_64_immediate_operand (new_rtx, mode))
			new_rtx = force_reg (mode, new_rtx);

		      new_rtx
		        = gen_rtx_PLUS (mode, force_reg (mode, base), new_rtx);
		    }
		  else
		    new_rtx = plus_constant (mode, base, INTVAL (new_rtx));
		}
	      else
		{
		  /* For %rip addressing, we have to use
		     just disp32, not base nor index.  */
		  if (TARGET_64BIT
		      && (GET_CODE (base) == SYMBOL_REF
			  || GET_CODE (base) == LABEL_REF))
		    base = force_reg (mode, base);
		  if (GET_CODE (new_rtx) == PLUS
		      && CONSTANT_P (XEXP (new_rtx, 1)))
		    {
		      base = gen_rtx_PLUS (mode, base, XEXP (new_rtx, 0));
		      new_rtx = XEXP (new_rtx, 1);
		    }
		  new_rtx = gen_rtx_PLUS (mode, base, new_rtx);
		}
	    }
	}
    }
  return new_rtx;
}

/* Load the thread pointer.  If TO_REG is true, force it into a register.  */

static rtx
get_thread_pointer (machine_mode tp_mode, bool to_reg)
{
  rtx tp = gen_rtx_UNSPEC (ptr_mode, gen_rtvec (1, const0_rtx), UNSPEC_TP);

  if (GET_MODE (tp) != tp_mode)
    {
      gcc_assert (GET_MODE (tp) == SImode);
      gcc_assert (tp_mode == DImode);

      tp = gen_rtx_ZERO_EXTEND (tp_mode, tp);
    }

  if (to_reg)
    tp = copy_to_mode_reg (tp_mode, tp);

  return tp;
}

/* Construct the SYMBOL_REF for the tls_get_addr function.  */

static GTY(()) rtx ix86_tls_symbol;

static rtx
ix86_tls_get_addr (void)
{
  if (!ix86_tls_symbol)
    {
      const char *sym
	= ((TARGET_ANY_GNU_TLS && !TARGET_64BIT)
	   ? "___tls_get_addr" : "__tls_get_addr");

      ix86_tls_symbol = gen_rtx_SYMBOL_REF (Pmode, sym);
    }

  if (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF)
    {
      rtx unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, ix86_tls_symbol),
				   UNSPEC_PLTOFF);
      return gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
			   gen_rtx_CONST (Pmode, unspec));
    }

  return ix86_tls_symbol;
}

/* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol.  */

static GTY(()) rtx ix86_tls_module_base_symbol;

rtx
ix86_tls_module_base (void)
{
  if (!ix86_tls_module_base_symbol)
    {
      ix86_tls_module_base_symbol
	= gen_rtx_SYMBOL_REF (ptr_mode, "_TLS_MODULE_BASE_");

      SYMBOL_REF_FLAGS (ix86_tls_module_base_symbol)
	|= TLS_MODEL_GLOBAL_DYNAMIC << SYMBOL_FLAG_TLS_SHIFT;
    }

  return ix86_tls_module_base_symbol;
}

/* A subroutine of ix86_legitimize_address and ix86_expand_move.  FOR_MOV is
   false if we expect this to be used for a memory address and true if
   we expect to load the address into a register.  */

rtx
legitimize_tls_address (rtx x, enum tls_model model, bool for_mov)
{
  rtx dest, base, off;
  rtx pic = NULL_RTX, tp = NULL_RTX;
  machine_mode tp_mode = Pmode;
  int type;

  /* Fall back to global dynamic model if tool chain cannot support local
     dynamic.  */
  if (TARGET_SUN_TLS && !TARGET_64BIT
      && !HAVE_AS_IX86_TLSLDMPLT && !HAVE_AS_IX86_TLSLDM
      && model == TLS_MODEL_LOCAL_DYNAMIC)
    model = TLS_MODEL_GLOBAL_DYNAMIC;

  switch (model)
    {
    case TLS_MODEL_GLOBAL_DYNAMIC:
      if (!TARGET_64BIT)
	{
	  if (flag_pic && !TARGET_PECOFF)
	    pic = pic_offset_table_rtx;
	  else
	    {
	      pic = gen_reg_rtx (Pmode);
	      emit_insn (gen_set_got (pic));
	    }
	}

      if (TARGET_GNU2_TLS)
	{
	  dest = gen_reg_rtx (ptr_mode);
	  if (TARGET_64BIT)
	    emit_insn (gen_tls_dynamic_gnu2_64 (ptr_mode, dest, x));
	  else
	    emit_insn (gen_tls_dynamic_gnu2_32 (dest, x, pic));

	  tp = get_thread_pointer (ptr_mode, true);
	  dest = gen_rtx_PLUS (ptr_mode, tp, dest);
	  if (GET_MODE (dest) != Pmode)
	     dest = gen_rtx_ZERO_EXTEND (Pmode, dest);
	  dest = force_reg (Pmode, dest);

	  if (GET_MODE (x) != Pmode)
	    x = gen_rtx_ZERO_EXTEND (Pmode, x);

	  set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
	}
      else
	{
	  rtx caddr = ix86_tls_get_addr ();

	  dest = gen_reg_rtx (Pmode);
	  if (TARGET_64BIT)
	    {
	      rtx rax = gen_rtx_REG (Pmode, AX_REG);
	      rtx_insn *insns;

	      start_sequence ();
	      emit_call_insn
		(gen_tls_global_dynamic_64 (Pmode, rax, x, caddr));
	      insns = get_insns ();
	      end_sequence ();

	      if (GET_MODE (x) != Pmode)
		x = gen_rtx_ZERO_EXTEND (Pmode, x);

	      RTL_CONST_CALL_P (insns) = 1;
	      emit_libcall_block (insns, dest, rax, x);
	    }
	  else
	    emit_insn (gen_tls_global_dynamic_32 (dest, x, pic, caddr));
	}
      break;

    case TLS_MODEL_LOCAL_DYNAMIC:
      if (!TARGET_64BIT)
	{
	  if (flag_pic)
	    pic = pic_offset_table_rtx;
	  else
	    {
	      pic = gen_reg_rtx (Pmode);
	      emit_insn (gen_set_got (pic));
	    }
	}

      if (TARGET_GNU2_TLS)
	{
	  rtx tmp = ix86_tls_module_base ();

	  base = gen_reg_rtx (ptr_mode);
	  if (TARGET_64BIT)
	    emit_insn (gen_tls_dynamic_gnu2_64 (ptr_mode, base, tmp));
	  else
	    emit_insn (gen_tls_dynamic_gnu2_32 (base, tmp, pic));

	  tp = get_thread_pointer (ptr_mode, true);
	  if (GET_MODE (base) != Pmode)
	    base = gen_rtx_ZERO_EXTEND (Pmode, base);
	  base = force_reg (Pmode, base);
	}
      else
	{
	  rtx caddr = ix86_tls_get_addr ();

	  base = gen_reg_rtx (Pmode);
	  if (TARGET_64BIT)
	    {
	      rtx rax = gen_rtx_REG (Pmode, AX_REG);
	      rtx_insn *insns;
	      rtx eqv;

	      start_sequence ();
	      emit_call_insn
		(gen_tls_local_dynamic_base_64 (Pmode, rax, caddr));
	      insns = get_insns ();
	      end_sequence ();

	      /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
		 share the LD_BASE result with other LD model accesses.  */
	      eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
				    UNSPEC_TLS_LD_BASE);

	      RTL_CONST_CALL_P (insns) = 1;
	      emit_libcall_block (insns, base, rax, eqv);
	    }
	  else
	    emit_insn (gen_tls_local_dynamic_base_32 (base, pic, caddr));
	}

      off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), UNSPEC_DTPOFF);
      off = gen_rtx_CONST (Pmode, off);

      dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, base, off));

      if (TARGET_GNU2_TLS)
	{
	  if (GET_MODE (tp) != Pmode)
	    {
	      dest = lowpart_subreg (ptr_mode, dest, Pmode);
	      dest = gen_rtx_PLUS (ptr_mode, tp, dest);
	      dest = gen_rtx_ZERO_EXTEND (Pmode, dest);
	    }
	  else
	    dest = gen_rtx_PLUS (Pmode, tp, dest);
	  dest = force_reg (Pmode, dest);

	  if (GET_MODE (x) != Pmode)
	    x = gen_rtx_ZERO_EXTEND (Pmode, x);

	  set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
	}
      break;

    case TLS_MODEL_INITIAL_EXEC:
      if (TARGET_64BIT)
	{
	  if (TARGET_SUN_TLS && !TARGET_X32)
	    {
	      /* The Sun linker took the AMD64 TLS spec literally
		 and can only handle %rax as destination of the
		 initial executable code sequence.  */

	      dest = gen_reg_rtx (DImode);
	      emit_insn (gen_tls_initial_exec_64_sun (dest, x));
	      return dest;
	    }

	  /* Generate DImode references to avoid %fs:(%reg32)
	     problems and linker IE->LE relaxation bug.  */
	  tp_mode = DImode;
	  pic = NULL;
	  type = UNSPEC_GOTNTPOFF;
	}
      else if (flag_pic)
	{
	  pic = pic_offset_table_rtx;
	  type = TARGET_ANY_GNU_TLS ? UNSPEC_GOTNTPOFF : UNSPEC_GOTTPOFF;
	}
      else if (!TARGET_ANY_GNU_TLS)
	{
	  pic = gen_reg_rtx (Pmode);
	  emit_insn (gen_set_got (pic));
	  type = UNSPEC_GOTTPOFF;
	}
      else
	{
	  pic = NULL;
	  type = UNSPEC_INDNTPOFF;
	}

      off = gen_rtx_UNSPEC (tp_mode, gen_rtvec (1, x), type);
      off = gen_rtx_CONST (tp_mode, off);
      if (pic)
	off = gen_rtx_PLUS (tp_mode, pic, off);
      off = gen_const_mem (tp_mode, off);
      set_mem_alias_set (off, ix86_GOT_alias_set ());

      if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
	{
	  base = get_thread_pointer (tp_mode,
				     for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
	  off = force_reg (tp_mode, off);
	  dest = gen_rtx_PLUS (tp_mode, base, off);
	  if (tp_mode != Pmode)
	    dest = convert_to_mode (Pmode, dest, 1);
	}
      else
	{
	  base = get_thread_pointer (Pmode, true);
	  dest = gen_reg_rtx (Pmode);
	  emit_insn (gen_sub3_insn (dest, base, off));
	}
      break;

    case TLS_MODEL_LOCAL_EXEC:
      off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
			    (TARGET_64BIT || TARGET_ANY_GNU_TLS)
			    ? UNSPEC_NTPOFF : UNSPEC_TPOFF);
      off = gen_rtx_CONST (Pmode, off);

      if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
	{
	  base = get_thread_pointer (Pmode,
				     for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
	  return gen_rtx_PLUS (Pmode, base, off);
	}
      else
	{
	  base = get_thread_pointer (Pmode, true);
	  dest = gen_reg_rtx (Pmode);
	  emit_insn (gen_sub3_insn (dest, base, off));
	}
      break;

    default:
      gcc_unreachable ();
    }

  return dest;
}

/* Return true if the TLS address requires insn using integer registers.
   It's used to prevent KMOV/VMOV in TLS code sequences which require integer
   MOV instructions, refer to PR103275.  */
bool
ix86_gpr_tls_address_pattern_p (rtx mem)
{
  gcc_assert (MEM_P (mem));

  rtx addr = XEXP (mem, 0);
  subrtx_var_iterator::array_type array;
  FOR_EACH_SUBRTX_VAR (iter, array, addr, ALL)
    {
      rtx op = *iter;
      if (GET_CODE (op) == UNSPEC)
	switch (XINT (op, 1))
	  {
	  case UNSPEC_GOTNTPOFF:
	    return true;
	  case UNSPEC_TPOFF:
	    if (!TARGET_64BIT)
	      return true;
	    break;
	  default:
	    break;
	  }
    }

  return false;
}

/* Return true if OP refers to a TLS address.  */
bool
ix86_tls_address_pattern_p (rtx op)
{
  subrtx_var_iterator::array_type array;
  FOR_EACH_SUBRTX_VAR (iter, array, op, ALL)
    {
      rtx op = *iter;
      if (MEM_P (op))
	{
	  rtx *x = &XEXP (op, 0);
	  while (GET_CODE (*x) == PLUS)
	    {
	      int i;
	      for (i = 0; i < 2; i++)
		{
		  rtx u = XEXP (*x, i);
		  if (GET_CODE (u) == ZERO_EXTEND)
		    u = XEXP (u, 0);
		  if (GET_CODE (u) == UNSPEC
		      && XINT (u, 1) == UNSPEC_TP)
		    return true;
		}
	      x = &XEXP (*x, 0);
	    }

	  iter.skip_subrtxes ();
	}
    }

  return false;
}

/* Rewrite *LOC so that it refers to a default TLS address space.  */
void
ix86_rewrite_tls_address_1 (rtx *loc)
{
  subrtx_ptr_iterator::array_type array;
  FOR_EACH_SUBRTX_PTR (iter, array, loc, ALL)
    {
      rtx *loc = *iter;
      if (MEM_P (*loc))
	{
	  rtx addr = XEXP (*loc, 0);
	  rtx *x = &addr;
	  while (GET_CODE (*x) == PLUS)
	    {
	      int i;
	      for (i = 0; i < 2; i++)
		{
		  rtx u = XEXP (*x, i);
		  if (GET_CODE (u) == ZERO_EXTEND)
		    u = XEXP (u, 0);
		  if (GET_CODE (u) == UNSPEC
		      && XINT (u, 1) == UNSPEC_TP)
		    {
		      addr_space_t as = DEFAULT_TLS_SEG_REG;

		      *x = XEXP (*x, 1 - i);

		      *loc = replace_equiv_address_nv (*loc, addr, true);
		      set_mem_addr_space (*loc, as);
		      return;
		    }
		}
	      x = &XEXP (*x, 0);
	    }

	  iter.skip_subrtxes ();
	}
    }
}

/* Rewrite instruction pattern involvning TLS address
   so that it refers to a default TLS address space.  */
rtx
ix86_rewrite_tls_address (rtx pattern)
{
  pattern = copy_insn (pattern);
  ix86_rewrite_tls_address_1 (&pattern);
  return pattern;
}

/* Create or return the unique __imp_DECL dllimport symbol corresponding
   to symbol DECL if BEIMPORT is true.  Otherwise create or return the
   unique refptr-DECL symbol corresponding to symbol DECL.  */

struct dllimport_hasher : ggc_cache_ptr_hash<tree_map>
{
  static inline hashval_t hash (tree_map *m) { return m->hash; }
  static inline bool
  equal (tree_map *a, tree_map *b)
  {
    return a->base.from == b->base.from;
  }

  static int
  keep_cache_entry (tree_map *&m)
  {
    return ggc_marked_p (m->base.from);
  }
};

static GTY((cache)) hash_table<dllimport_hasher> *dllimport_map;

static tree
get_dllimport_decl (tree decl, bool beimport)
{
  struct tree_map *h, in;
  const char *name;
  const char *prefix;
  size_t namelen, prefixlen;
  char *imp_name;
  tree to;
  rtx rtl;

  if (!dllimport_map)
    dllimport_map = hash_table<dllimport_hasher>::create_ggc (512);

  in.hash = htab_hash_pointer (decl);
  in.base.from = decl;
  tree_map **loc = dllimport_map->find_slot_with_hash (&in, in.hash, INSERT);
  h = *loc;
  if (h)
    return h->to;

  *loc = h = ggc_alloc<tree_map> ();
  h->hash = in.hash;
  h->base.from = decl;
  h->to = to = build_decl (DECL_SOURCE_LOCATION (decl),
			   VAR_DECL, NULL, ptr_type_node);
  DECL_ARTIFICIAL (to) = 1;
  DECL_IGNORED_P (to) = 1;
  DECL_EXTERNAL (to) = 1;
  TREE_READONLY (to) = 1;

  name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
  name = targetm.strip_name_encoding (name);
  if (beimport)
    prefix = name[0] == FASTCALL_PREFIX || user_label_prefix[0] == 0
      ? "*__imp_" : "*__imp__";
  else
    prefix = user_label_prefix[0] == 0 ? "*.refptr." : "*refptr.";
  namelen = strlen (name);
  prefixlen = strlen (prefix);
  imp_name = (char *) alloca (namelen + prefixlen + 1);
  memcpy (imp_name, prefix, prefixlen);
  memcpy (imp_name + prefixlen, name, namelen + 1);

  name = ggc_alloc_string (imp_name, namelen + prefixlen);
  rtl = gen_rtx_SYMBOL_REF (Pmode, name);
  SET_SYMBOL_REF_DECL (rtl, to);
  SYMBOL_REF_FLAGS (rtl) = SYMBOL_FLAG_LOCAL | SYMBOL_FLAG_STUBVAR;
  if (!beimport)
    {
      SYMBOL_REF_FLAGS (rtl) |= SYMBOL_FLAG_EXTERNAL;
#ifdef SUB_TARGET_RECORD_STUB
      SUB_TARGET_RECORD_STUB (name);
#endif
    }      

  rtl = gen_const_mem (Pmode, rtl);
  set_mem_alias_set (rtl, ix86_GOT_alias_set ());

  SET_DECL_RTL (to, rtl);
  SET_DECL_ASSEMBLER_NAME (to, get_identifier (name));

  return to;
}

/* Expand SYMBOL into its corresponding far-address symbol.
   WANT_REG is true if we require the result be a register.  */

static rtx
legitimize_pe_coff_extern_decl (rtx symbol, bool want_reg)
{
  tree imp_decl;
  rtx x;

  gcc_assert (SYMBOL_REF_DECL (symbol));
  imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol), false);

  x = DECL_RTL (imp_decl);
  if (want_reg)
    x = force_reg (Pmode, x);
  return x;
}

/* Expand SYMBOL into its corresponding dllimport symbol.  WANT_REG is
   true if we require the result be a register.  */

static rtx
legitimize_dllimport_symbol (rtx symbol, bool want_reg)
{
  tree imp_decl;
  rtx x;

  gcc_assert (SYMBOL_REF_DECL (symbol));
  imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol), true);

  x = DECL_RTL (imp_decl);
  if (want_reg)
    x = force_reg (Pmode, x);
  return x;
}

/* Expand SYMBOL into its corresponding dllimport or refptr symbol.  WANT_REG 
   is true if we require the result be a register.  */

rtx
legitimize_pe_coff_symbol (rtx addr, bool inreg)
{
  if (!TARGET_PECOFF)
    return NULL_RTX;

  if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
    {
      if (GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_DLLIMPORT_P (addr))
	return legitimize_dllimport_symbol (addr, inreg);
      if (GET_CODE (addr) == CONST
	  && GET_CODE (XEXP (addr, 0)) == PLUS
	  && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
	  && SYMBOL_REF_DLLIMPORT_P (XEXP (XEXP (addr, 0), 0)))
	{
	  rtx t = legitimize_dllimport_symbol (XEXP (XEXP (addr, 0), 0), inreg);
	  return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
	}
    }

  if (ix86_cmodel != CM_LARGE_PIC && ix86_cmodel != CM_MEDIUM_PIC)
    return NULL_RTX;
  if (GET_CODE (addr) == SYMBOL_REF
      && !is_imported_p (addr)
      && SYMBOL_REF_EXTERNAL_P (addr)
      && SYMBOL_REF_DECL (addr))
    return legitimize_pe_coff_extern_decl (addr, inreg);

  if (GET_CODE (addr) == CONST
      && GET_CODE (XEXP (addr, 0)) == PLUS
      && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
      && !is_imported_p (XEXP (XEXP (addr, 0), 0))
      && SYMBOL_REF_EXTERNAL_P (XEXP (XEXP (addr, 0), 0))
      && SYMBOL_REF_DECL (XEXP (XEXP (addr, 0), 0)))
    {
      rtx t = legitimize_pe_coff_extern_decl (XEXP (XEXP (addr, 0), 0), inreg);
      return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
    }
  return NULL_RTX;
}

/* Try machine-dependent ways of modifying an illegitimate address
   to be legitimate.  If we find one, return the new, valid address.
   This macro is used in only one place: `memory_address' in explow.cc.

   OLDX is the address as it was before break_out_memory_refs was called.
   In some cases it is useful to look at this to decide what needs to be done.

   It is always safe for this macro to do nothing.  It exists to recognize
   opportunities to optimize the output.

   For the 80386, we handle X+REG by loading X into a register R and
   using R+REG.  R will go in a general reg and indexing will be used.
   However, if REG is a broken-out memory address or multiplication,
   nothing needs to be done because REG can certainly go in a general reg.

   When -fpic is used, special handling is needed for symbolic references.
   See comments by legitimize_pic_address in i386.cc for details.  */

static rtx
ix86_legitimize_address (rtx x, rtx, machine_mode mode)
{
  bool changed = false;
  unsigned log;

  log = GET_CODE (x) == SYMBOL_REF ? SYMBOL_REF_TLS_MODEL (x) : 0;
  if (log)
    return legitimize_tls_address (x, (enum tls_model) log, false);
  if (GET_CODE (x) == CONST
      && GET_CODE (XEXP (x, 0)) == PLUS
      && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
      && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
    {
      rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
				      (enum tls_model) log, false);
      return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
    }

  if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
    {
      rtx tmp = legitimize_pe_coff_symbol (x, true);
      if (tmp)
        return tmp;
    }

  if (flag_pic && SYMBOLIC_CONST (x))
    return legitimize_pic_address (x, 0);

#if TARGET_MACHO
  if (MACHO_DYNAMIC_NO_PIC_P && SYMBOLIC_CONST (x))
    return machopic_indirect_data_reference (x, 0);
#endif

  /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
  if (GET_CODE (x) == ASHIFT
      && CONST_INT_P (XEXP (x, 1))
      && (unsigned HOST_WIDE_INT) INTVAL (XEXP (x, 1)) < 4)
    {
      changed = true;
      log = INTVAL (XEXP (x, 1));
      x = gen_rtx_MULT (Pmode, force_reg (Pmode, XEXP (x, 0)),
			GEN_INT (1 << log));
    }

  if (GET_CODE (x) == PLUS)
    {
      /* Canonicalize shifts by 0, 1, 2, 3 into multiply.  */

      if (GET_CODE (XEXP (x, 0)) == ASHIFT
	  && CONST_INT_P (XEXP (XEXP (x, 0), 1))
	  && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 0), 1)) < 4)
	{
	  changed = true;
	  log = INTVAL (XEXP (XEXP (x, 0), 1));
	  XEXP (x, 0) = gen_rtx_MULT (Pmode,
				      force_reg (Pmode, XEXP (XEXP (x, 0), 0)),
				      GEN_INT (1 << log));
	}

      if (GET_CODE (XEXP (x, 1)) == ASHIFT
	  && CONST_INT_P (XEXP (XEXP (x, 1), 1))
	  && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 1), 1)) < 4)
	{
	  changed = true;
	  log = INTVAL (XEXP (XEXP (x, 1), 1));
	  XEXP (x, 1) = gen_rtx_MULT (Pmode,
				      force_reg (Pmode, XEXP (XEXP (x, 1), 0)),
				      GEN_INT (1 << log));
	}

      /* Put multiply first if it isn't already.  */
      if (GET_CODE (XEXP (x, 1)) == MULT)
	{
	  std::swap (XEXP (x, 0), XEXP (x, 1));
	  changed = true;
	}

      /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
	 into (plus (plus (mult (reg) (const)) (reg)) (const)).  This can be
	 created by virtual register instantiation, register elimination, and
	 similar optimizations.  */
      if (GET_CODE (XEXP (x, 0)) == MULT && GET_CODE (XEXP (x, 1)) == PLUS)
	{
	  changed = true;
	  x = gen_rtx_PLUS (Pmode,
			    gen_rtx_PLUS (Pmode, XEXP (x, 0),
					  XEXP (XEXP (x, 1), 0)),
			    XEXP (XEXP (x, 1), 1));
	}

      /* Canonicalize
	 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
	 into (plus (plus (mult (reg) (const)) (reg)) (const)).  */
      else if (GET_CODE (x) == PLUS && GET_CODE (XEXP (x, 0)) == PLUS
	       && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
	       && GET_CODE (XEXP (XEXP (x, 0), 1)) == PLUS
	       && CONSTANT_P (XEXP (x, 1)))
	{
	  rtx constant;
	  rtx other = NULL_RTX;

	  if (CONST_INT_P (XEXP (x, 1)))
	    {
	      constant = XEXP (x, 1);
	      other = XEXP (XEXP (XEXP (x, 0), 1), 1);
	    }
	  else if (CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 1), 1)))
	    {
	      constant = XEXP (XEXP (XEXP (x, 0), 1), 1);
	      other = XEXP (x, 1);
	    }
	  else
	    constant = 0;

	  if (constant)
	    {
	      changed = true;
	      x = gen_rtx_PLUS (Pmode,
				gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
					      XEXP (XEXP (XEXP (x, 0), 1), 0)),
				plus_constant (Pmode, other,
					       INTVAL (constant)));
	    }
	}

      if (changed && ix86_legitimate_address_p (mode, x, false))
	return x;

      if (GET_CODE (XEXP (x, 0)) == MULT)
	{
	  changed = true;
	  XEXP (x, 0) = copy_addr_to_reg (XEXP (x, 0));
	}

      if (GET_CODE (XEXP (x, 1)) == MULT)
	{
	  changed = true;
	  XEXP (x, 1) = copy_addr_to_reg (XEXP (x, 1));
	}

      if (changed
	  && REG_P (XEXP (x, 1))
	  && REG_P (XEXP (x, 0)))
	return x;

      if (flag_pic && SYMBOLIC_CONST (XEXP (x, 1)))
	{
	  changed = true;
	  x = legitimize_pic_address (x, 0);
	}

      if (changed && ix86_legitimate_address_p (mode, x, false))
	return x;

      if (REG_P (XEXP (x, 0)))
	{
	  rtx temp = gen_reg_rtx (Pmode);
	  rtx val  = force_operand (XEXP (x, 1), temp);
	  if (val != temp)
	    {
	      val = convert_to_mode (Pmode, val, 1);
	      emit_move_insn (temp, val);
	    }

	  XEXP (x, 1) = temp;
	  return x;
	}

      else if (REG_P (XEXP (x, 1)))
	{
	  rtx temp = gen_reg_rtx (Pmode);
	  rtx val  = force_operand (XEXP (x, 0), temp);
	  if (val != temp)
	    {
	      val = convert_to_mode (Pmode, val, 1);
	      emit_move_insn (temp, val);
	    }

	  XEXP (x, 0) = temp;
	  return x;
	}
    }

  return x;
}

/* Print an integer constant expression in assembler syntax.  Addition
   and subtraction are the only arithmetic that may appear in these
   expressions.  FILE is the stdio stream to write to, X is the rtx, and
   CODE is the operand print code from the output string.  */

static void
output_pic_addr_const (FILE *file, rtx x, int code)
{
  char buf[256];

  switch (GET_CODE (x))
    {
    case PC:
      gcc_assert (flag_pic);
      putc ('.', file);
      break;

    case SYMBOL_REF:
      if (TARGET_64BIT || ! TARGET_MACHO_SYMBOL_STUBS)
	output_addr_const (file, x);
      else
	{
	  const char *name = XSTR (x, 0);

	  /* Mark the decl as referenced so that cgraph will
	     output the function.  */
	  if (SYMBOL_REF_DECL (x))
	    mark_decl_referenced (SYMBOL_REF_DECL (x));

#if TARGET_MACHO
	  if (MACHOPIC_INDIRECT
	      && machopic_classify_symbol (x) == MACHOPIC_UNDEFINED_FUNCTION)
	    name = machopic_indirection_name (x, /*stub_p=*/true);
#endif
	  assemble_name (file, name);
	}
      if (!TARGET_MACHO && !(TARGET_64BIT && TARGET_PECOFF)
	  && code == 'P' && ix86_call_use_plt_p (x))
	fputs ("@PLT", file);
      break;

    case LABEL_REF:
      x = XEXP (x, 0);
      /* FALLTHRU */
    case CODE_LABEL:
      ASM_GENERATE_INTERNAL_LABEL (buf, "L", CODE_LABEL_NUMBER (x));
      assemble_name (asm_out_file, buf);
      break;

    CASE_CONST_SCALAR_INT:
      output_addr_const (file, x);
      break;

    case CONST:
      /* This used to output parentheses around the expression,
	 but that does not work on the 386 (either ATT or BSD assembler).  */
      output_pic_addr_const (file, XEXP (x, 0), code);
      break;

    case CONST_DOUBLE:
      /* We can't handle floating point constants;
	 TARGET_PRINT_OPERAND must handle them.  */
      output_operand_lossage ("floating constant misused");
      break;

    case PLUS:
      /* Some assemblers need integer constants to appear first.  */
      if (CONST_INT_P (XEXP (x, 0)))
	{
	  output_pic_addr_const (file, XEXP (x, 0), code);
	  putc ('+', file);
	  output_pic_addr_const (file, XEXP (x, 1), code);
	}
      else
	{
	  gcc_assert (CONST_INT_P (XEXP (x, 1)));
	  output_pic_addr_const (file, XEXP (x, 1), code);
	  putc ('+', file);
	  output_pic_addr_const (file, XEXP (x, 0), code);
	}
      break;

    case MINUS:
      if (!TARGET_MACHO)
	putc (ASSEMBLER_DIALECT == ASM_INTEL ? '(' : '[', file);
      output_pic_addr_const (file, XEXP (x, 0), code);
      putc ('-', file);
      output_pic_addr_const (file, XEXP (x, 1), code);
      if (!TARGET_MACHO)
	putc (ASSEMBLER_DIALECT == ASM_INTEL ? ')' : ']', file);
      break;

    case UNSPEC:
      gcc_assert (XVECLEN (x, 0) == 1);
      output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
      switch (XINT (x, 1))
	{
	case UNSPEC_GOT:
	  fputs ("@GOT", file);
	  break;
	case UNSPEC_GOTOFF:
	  fputs ("@GOTOFF", file);
	  break;
	case UNSPEC_PLTOFF:
	  fputs ("@PLTOFF", file);
	  break;
	case UNSPEC_PCREL:
	  fputs (ASSEMBLER_DIALECT == ASM_ATT ?
		 "(%rip)" : "[rip]", file);
	  break;
	case UNSPEC_GOTPCREL:
	  fputs (ASSEMBLER_DIALECT == ASM_ATT ?
		 "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file);
	  break;
	case UNSPEC_GOTTPOFF:
	  /* FIXME: This might be @TPOFF in Sun ld too.  */
	  fputs ("@gottpoff", file);
	  break;
	case UNSPEC_TPOFF:
	  fputs ("@tpoff", file);
	  break;
	case UNSPEC_NTPOFF:
	  if (TARGET_64BIT)
	    fputs ("@tpoff", file);
	  else
	    fputs ("@ntpoff", file);
	  break;
	case UNSPEC_DTPOFF:
	  fputs ("@dtpoff", file);
	  break;
	case UNSPEC_GOTNTPOFF:
	  if (TARGET_64BIT)
	    fputs (ASSEMBLER_DIALECT == ASM_ATT ?
		   "@gottpoff(%rip)": "@gottpoff[rip]", file);
	  else
	    fputs ("@gotntpoff", file);
	  break;
	case UNSPEC_INDNTPOFF:
	  fputs ("@indntpoff", file);
	  break;
#if TARGET_MACHO
	case UNSPEC_MACHOPIC_OFFSET:
	  putc ('-', file);
	  machopic_output_function_base_name (file);
	  break;
#endif
	default:
	  output_operand_lossage ("invalid UNSPEC as operand");
	  break;
	}
       break;

    default:
      output_operand_lossage ("invalid expression as operand");
    }
}

/* This is called from dwarf2out.cc via TARGET_ASM_OUTPUT_DWARF_DTPREL.
   We need to emit DTP-relative relocations.  */

static void ATTRIBUTE_UNUSED
i386_output_dwarf_dtprel (FILE *file, int size, rtx x)
{
  fputs (ASM_LONG, file);
  output_addr_const (file, x);
  fputs ("@dtpoff", file);
  switch (size)
    {
    case 4:
      break;
    case 8:
      fputs (", 0", file);
      break;
    default:
      gcc_unreachable ();
   }
}

/* Return true if X is a representation of the PIC register.  This copes
   with calls from ix86_find_base_term, where the register might have
   been replaced by a cselib value.  */

static bool
ix86_pic_register_p (rtx x)
{
  if (GET_CODE (x) == VALUE && CSELIB_VAL_PTR (x))
    return (pic_offset_table_rtx
	    && rtx_equal_for_cselib_p (x, pic_offset_table_rtx));
  else if (GET_CODE (x) == UNSPEC && XINT (x, 1) == UNSPEC_SET_GOT)
    return true;
  else if (!REG_P (x))
    return false;
  else if (pic_offset_table_rtx)
    {
      if (REGNO (x) == REGNO (pic_offset_table_rtx))
	return true;
      if (HARD_REGISTER_P (x)
	  && !HARD_REGISTER_P (pic_offset_table_rtx)
	  && ORIGINAL_REGNO (x) == REGNO (pic_offset_table_rtx))
	return true;
      return false;
    }
  else
    return REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
}

/* Helper function for ix86_delegitimize_address.
   Attempt to delegitimize TLS local-exec accesses.  */

static rtx
ix86_delegitimize_tls_address (rtx orig_x)
{
  rtx x = orig_x, unspec;
  struct ix86_address addr;

  if (!TARGET_TLS_DIRECT_SEG_REFS)
    return orig_x;
  if (MEM_P (x))
    x = XEXP (x, 0);
  if (GET_CODE (x) != PLUS || GET_MODE (x) != Pmode)
    return orig_x;
  if (ix86_decompose_address (x, &addr) == 0
      || addr.seg != DEFAULT_TLS_SEG_REG
      || addr.disp == NULL_RTX
      || GET_CODE (addr.disp) != CONST)
    return orig_x;
  unspec = XEXP (addr.disp, 0);
  if (GET_CODE (unspec) == PLUS && CONST_INT_P (XEXP (unspec, 1)))
    unspec = XEXP (unspec, 0);
  if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
    return orig_x;
  x = XVECEXP (unspec, 0, 0);
  gcc_assert (GET_CODE (x) == SYMBOL_REF);
  if (unspec != XEXP (addr.disp, 0))
    x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
  if (addr.index)
    {
      rtx idx = addr.index;
      if (addr.scale != 1)
	idx = gen_rtx_MULT (Pmode, idx, GEN_INT (addr.scale));
      x = gen_rtx_PLUS (Pmode, idx, x);
    }
  if (addr.base)
    x = gen_rtx_PLUS (Pmode, addr.base, x);
  if (MEM_P (orig_x))
    x = replace_equiv_address_nv (orig_x, x);
  return x;
}

/* In the name of slightly smaller debug output, and to cater to
   general assembler lossage, recognize PIC+GOTOFF and turn it back
   into a direct symbol reference.

   On Darwin, this is necessary to avoid a crash, because Darwin
   has a different PIC label for each routine but the DWARF debugging
   information is not associated with any particular routine, so it's
   necessary to remove references to the PIC label from RTL stored by
   the DWARF output code.

   This helper is used in the normal ix86_delegitimize_address
   entrypoint (e.g. used in the target delegitimization hook) and
   in ix86_find_base_term.  As compile time memory optimization, we
   avoid allocating rtxes that will not change anything on the outcome
   of the callers (find_base_value and find_base_term).  */

static inline rtx
ix86_delegitimize_address_1 (rtx x, bool base_term_p)
{
  rtx orig_x = delegitimize_mem_from_attrs (x);
  /* addend is NULL or some rtx if x is something+GOTOFF where
     something doesn't include the PIC register.  */
  rtx addend = NULL_RTX;
  /* reg_addend is NULL or a multiple of some register.  */
  rtx reg_addend = NULL_RTX;
  /* const_addend is NULL or a const_int.  */
  rtx const_addend = NULL_RTX;
  /* This is the result, or NULL.  */
  rtx result = NULL_RTX;

  x = orig_x;

  if (MEM_P (x))
    x = XEXP (x, 0);

  if (TARGET_64BIT)
    {
      if (GET_CODE (x) == CONST
          && GET_CODE (XEXP (x, 0)) == PLUS
          && GET_MODE (XEXP (x, 0)) == Pmode
          && CONST_INT_P (XEXP (XEXP (x, 0), 1))
          && GET_CODE (XEXP (XEXP (x, 0), 0)) == UNSPEC
          && XINT (XEXP (XEXP (x, 0), 0), 1) == UNSPEC_PCREL)
        {
	  /* find_base_{value,term} only care about MEMs with arg_pointer_rtx
	     base.  A CONST can't be arg_pointer_rtx based.  */
	  if (base_term_p && MEM_P (orig_x))
	    return orig_x;
	  rtx x2 = XVECEXP (XEXP (XEXP (x, 0), 0), 0, 0);
	  x = gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 1), x2);
	  if (MEM_P (orig_x))
	    x = replace_equiv_address_nv (orig_x, x);
	  return x;
	}

      if (GET_CODE (x) == CONST
	  && GET_CODE (XEXP (x, 0)) == UNSPEC
	  && (XINT (XEXP (x, 0), 1) == UNSPEC_GOTPCREL
	      || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL)
	  && (MEM_P (orig_x) || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL))
	{
	  x = XVECEXP (XEXP (x, 0), 0, 0);
	  if (GET_MODE (orig_x) != GET_MODE (x) && MEM_P (orig_x))
	    {
	      x = lowpart_subreg (GET_MODE (orig_x), x, GET_MODE (x));
	      if (x == NULL_RTX)
		return orig_x;
	    }
	  return x;
	}

      if (ix86_cmodel != CM_MEDIUM_PIC && ix86_cmodel != CM_LARGE_PIC)
	return ix86_delegitimize_tls_address (orig_x);

      /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
	 and -mcmodel=medium -fpic.  */
    }

  if (GET_CODE (x) != PLUS
      || GET_CODE (XEXP (x, 1)) != CONST)
    return ix86_delegitimize_tls_address (orig_x);

  if (ix86_pic_register_p (XEXP (x, 0)))
    /* %ebx + GOT/GOTOFF */
    ;
  else if (GET_CODE (XEXP (x, 0)) == PLUS)
    {
      /* %ebx + %reg * scale + GOT/GOTOFF */
      reg_addend = XEXP (x, 0);
      if (ix86_pic_register_p (XEXP (reg_addend, 0)))
	reg_addend = XEXP (reg_addend, 1);
      else if (ix86_pic_register_p (XEXP (reg_addend, 1)))
	reg_addend = XEXP (reg_addend, 0);
      else
	{
	  reg_addend = NULL_RTX;
	  addend = XEXP (x, 0);
	}
    }
  else
    addend = XEXP (x, 0);

  x = XEXP (XEXP (x, 1), 0);
  if (GET_CODE (x) == PLUS
      && CONST_INT_P (XEXP (x, 1)))
    {
      const_addend = XEXP (x, 1);
      x = XEXP (x, 0);
    }

  if (GET_CODE (x) == UNSPEC
      && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x) && !addend)
	  || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x))
	  || (XINT (x, 1) == UNSPEC_PLTOFF && ix86_cmodel == CM_LARGE_PIC
	      && !MEM_P (orig_x) && !addend)))
    result = XVECEXP (x, 0, 0);

  if (!TARGET_64BIT && TARGET_MACHO && darwin_local_data_pic (x)
      && !MEM_P (orig_x))
    result = XVECEXP (x, 0, 0);

  if (! result)
    return ix86_delegitimize_tls_address (orig_x);

  /* For (PLUS something CONST_INT) both find_base_{value,term} just
     recurse on the first operand.  */
  if (const_addend && !base_term_p)
    result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
  if (reg_addend)
    result = gen_rtx_PLUS (Pmode, reg_addend, result);
  if (addend)
    {
      /* If the rest of original X doesn't involve the PIC register, add
	 addend and subtract pic_offset_table_rtx.  This can happen e.g.
	 for code like:
	 leal (%ebx, %ecx, 4), %ecx
	 ...
	 movl foo@GOTOFF(%ecx), %edx
	 in which case we return (%ecx - %ebx) + foo
	 or (%ecx - _GLOBAL_OFFSET_TABLE_) + foo if pseudo_pic_reg
	 and reload has completed.  Don't do the latter for debug,
	 as _GLOBAL_OFFSET_TABLE_ can't be expressed in the assembly.  */
      if (pic_offset_table_rtx
	  && (!reload_completed || !ix86_use_pseudo_pic_reg ()))
        result = gen_rtx_PLUS (Pmode, gen_rtx_MINUS (Pmode, copy_rtx (addend),
						     pic_offset_table_rtx),
			       result);
      else if (base_term_p
	       && pic_offset_table_rtx
	       && !TARGET_MACHO
	       && !TARGET_VXWORKS_RTP)
	{
	  rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
	  tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
	  result = gen_rtx_PLUS (Pmode, tmp, result);
	}
      else
	return orig_x;
    }
  if (GET_MODE (orig_x) != Pmode && MEM_P (orig_x))
    {
      result = lowpart_subreg (GET_MODE (orig_x), result, Pmode);
      if (result == NULL_RTX)
	return orig_x;
    }
  return result;
}

/* The normal instantiation of the above template.  */

static rtx
ix86_delegitimize_address (rtx x)
{
  return ix86_delegitimize_address_1 (x, false);
}

/* If X is a machine specific address (i.e. a symbol or label being
   referenced as a displacement from the GOT implemented using an
   UNSPEC), then return the base term.  Otherwise return X.  */

rtx
ix86_find_base_term (rtx x)
{
  rtx term;

  if (TARGET_64BIT)
    {
      if (GET_CODE (x) != CONST)
	return x;
      term = XEXP (x, 0);
      if (GET_CODE (term) == PLUS
	  && CONST_INT_P (XEXP (term, 1)))
	term = XEXP (term, 0);
      if (GET_CODE (term) != UNSPEC
	  || (XINT (term, 1) != UNSPEC_GOTPCREL
	      && XINT (term, 1) != UNSPEC_PCREL))
	return x;

      return XVECEXP (term, 0, 0);
    }

  return ix86_delegitimize_address_1 (x, true);
}

/* Return true if X shouldn't be emitted into the debug info.
   Disallow UNSPECs other than @gotoff - we can't emit _GLOBAL_OFFSET_TABLE_
   symbol easily into the .debug_info section, so we need not to
   delegitimize, but instead assemble as @gotoff.
   Disallow _GLOBAL_OFFSET_TABLE_ SYMBOL_REF - the assembler magically
   assembles that as _GLOBAL_OFFSET_TABLE_-. expression.  */

static bool
ix86_const_not_ok_for_debug_p (rtx x)
{
  if (GET_CODE (x) == UNSPEC && XINT (x, 1) != UNSPEC_GOTOFF)
    return true;

  if (SYMBOL_REF_P (x) && strcmp (XSTR (x, 0), GOT_SYMBOL_NAME) == 0)
    return true;

  return false;
}

static void
put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
		    bool fp, FILE *file)
{
  const char *suffix;

  if (mode == CCFPmode)
    {
      code = ix86_fp_compare_code_to_integer (code);
      mode = CCmode;
    }
  if (reverse)
    code = reverse_condition (code);

  switch (code)
    {
    case EQ:
      gcc_assert (mode != CCGZmode);
      switch (mode)
	{
	case E_CCAmode:
	  suffix = "a";
	  break;
	case E_CCCmode:
	  suffix = "c";
	  break;
	case E_CCOmode:
	  suffix = "o";
	  break;
	case E_CCPmode:
	  suffix = "p";
	  break;
	case E_CCSmode:
	  suffix = "s";
	  break;
	default:
	  suffix = "e";
	  break;
	}
      break;
    case NE:
      gcc_assert (mode != CCGZmode);
      switch (mode)
	{
	case E_CCAmode:
	  suffix = "na";
	  break;
	case E_CCCmode:
	  suffix = "nc";
	  break;
	case E_CCOmode:
	  suffix = "no";
	  break;
	case E_CCPmode:
	  suffix = "np";
	  break;
	case E_CCSmode:
	  suffix = "ns";
	  break;
	default:
	  suffix = "ne";
	  break;
	}
      break;
    case GT:
      gcc_assert (mode == CCmode || mode == CCNOmode || mode == CCGCmode);
      suffix = "g";
      break;
    case GTU:
      /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
	 Those same assemblers have the same but opposite lossage on cmov.  */
      if (mode == CCmode)
	suffix = fp ? "nbe" : "a";
      else
	gcc_unreachable ();
      break;
    case LT:
      switch (mode)
	{
	case E_CCNOmode:
	case E_CCGOCmode:
	  suffix = "s";
	  break;

	case E_CCmode:
	case E_CCGCmode:
	case E_CCGZmode:
	  suffix = "l";
	  break;

	default:
	  gcc_unreachable ();
	}
      break;
    case LTU:
      if (mode == CCmode || mode == CCGZmode)
	suffix = "b";
      else if (mode == CCCmode)
	suffix = fp ? "b" : "c";
      else
	gcc_unreachable ();
      break;
    case GE:
      switch (mode)
	{
	case E_CCNOmode:
	case E_CCGOCmode:
	  suffix = "ns";
	  break;

	case E_CCmode:
	case E_CCGCmode:
	case E_CCGZmode:
	  suffix = "ge";
	  break;

	default:
	  gcc_unreachable ();
	}
      break;
    case GEU:
      if (mode == CCmode || mode == CCGZmode)
	suffix = "nb";
      else if (mode == CCCmode)
	suffix = fp ? "nb" : "nc";
      else
	gcc_unreachable ();
      break;
    case LE:
      gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
      suffix = "le";
      break;
    case LEU:
      if (mode == CCmode)
	suffix = "be";
      else
	gcc_unreachable ();
      break;
    case UNORDERED:
      suffix = fp ? "u" : "p";
      break;
    case ORDERED:
      suffix = fp ? "nu" : "np";
      break;
    default:
      gcc_unreachable ();
    }
  fputs (suffix, file);
}

/* Print the name of register X to FILE based on its machine mode and number.
   If CODE is 'w', pretend the mode is HImode.
   If CODE is 'b', pretend the mode is QImode.
   If CODE is 'k', pretend the mode is SImode.
   If CODE is 'q', pretend the mode is DImode.
   If CODE is 'x', pretend the mode is V4SFmode.
   If CODE is 't', pretend the mode is V8SFmode.
   If CODE is 'g', pretend the mode is V16SFmode.
   If CODE is 'h', pretend the reg is the 'high' byte register.
   If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
   If CODE is 'd', duplicate the operand for AVX instruction.
   If CODE is 'V', print naked full integer register name without %.
 */

void
print_reg (rtx x, int code, FILE *file)
{
  const char *reg;
  int msize;
  unsigned int regno;
  bool duplicated;

  if (ASSEMBLER_DIALECT == ASM_ATT && code != 'V')
    putc ('%', file);

  if (x == pc_rtx)
    {
      gcc_assert (TARGET_64BIT);
      fputs ("rip", file);
      return;
    }

  if (code == 'y' && STACK_TOP_P (x))
    {
      fputs ("st(0)", file);
      return;
    }

  if (code == 'w')
    msize = 2;
  else if (code == 'b')
    msize = 1;
  else if (code == 'k')
    msize = 4;
  else if (code == 'q')
    msize = 8;
  else if (code == 'h')
    msize = 0;
  else if (code == 'x')
    msize = 16;
  else if (code == 't')
    msize = 32;
  else if (code == 'g')
    msize = 64;
  else
    msize = GET_MODE_SIZE (GET_MODE (x));

  regno = REGNO (x);

  if (regno == ARG_POINTER_REGNUM
      || regno == FRAME_POINTER_REGNUM
      || regno == FPSR_REG)
    {
      output_operand_lossage
	("invalid use of register '%s'", reg_names[regno]);
      return;
    }
  else if (regno == FLAGS_REG)
    {
      output_operand_lossage ("invalid use of asm flag output");
      return;
    }

  if (code == 'V')
    {
      if (GENERAL_REGNO_P (regno))
	msize = GET_MODE_SIZE (word_mode);
      else
	error ("%<V%> modifier on non-integer register");
    }

  duplicated = code == 'd' && TARGET_AVX;

  switch (msize)
    {
    case 16:
    case 12:
    case 8:
      if (GENERAL_REGNO_P (regno) && msize > GET_MODE_SIZE (word_mode))
	warning (0, "unsupported size for integer register");
      /* FALLTHRU */
    case 4:
      if (LEGACY_INT_REGNO_P (regno))
	putc (msize > 4 && TARGET_64BIT ? 'r' : 'e', file);
      /* FALLTHRU */
    case 2:
    normal:
      reg = hi_reg_name[regno];
      break;
    case 1:
      if (regno >= ARRAY_SIZE (qi_reg_name))
	goto normal;
      if (!ANY_QI_REGNO_P (regno))
	error ("unsupported size for integer register");
      reg = qi_reg_name[regno];
      break;
    case 0:
      if (regno >= ARRAY_SIZE (qi_high_reg_name))
	goto normal;
      reg = qi_high_reg_name[regno];
      break;
    case 32:
    case 64:
      if (SSE_REGNO_P (regno))
	{
	  gcc_assert (!duplicated);
	  putc (msize == 32 ? 'y' : 'z', file);
	  reg = hi_reg_name[regno] + 1;
	  break;
	}
      goto normal;
    default:
      gcc_unreachable ();
    }

  fputs (reg, file);

  /* Irritatingly, AMD extended registers use
     different naming convention: "r%d[bwd]"  */
  if (REX_INT_REGNO_P (regno) || REX2_INT_REGNO_P (regno))
    {
      gcc_assert (TARGET_64BIT);
      switch (msize)
	{
	  case 0:
	    error ("extended registers have no high halves");
	    break;
	  case 1:
	    putc ('b', file);
	    break;
	  case 2:
	    putc ('w', file);
	    break;
	  case 4:
	    putc ('d', file);
	    break;
	  case 8:
	    /* no suffix */
	    break;
	  default:
	    error ("unsupported operand size for extended register");
	    break;
	}
      return;
    }

  if (duplicated)
    {
      if (ASSEMBLER_DIALECT == ASM_ATT)
	fprintf (file, ", %%%s", reg);
      else
	fprintf (file, ", %s", reg);
    }
}

/* Meaning of CODE:
   L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
   C -- print opcode suffix for set/cmov insn.
   c -- like C, but print reversed condition
   F,f -- likewise, but for floating-point.
   O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
	otherwise nothing
   R -- print embedded rounding and sae.
   r -- print only sae.
   z -- print the opcode suffix for the size of the current operand.
   Z -- likewise, with special suffixes for x87 instructions.
   * -- print a star (in certain assembler syntax)
   A -- print an absolute memory reference.
   E -- print address with DImode register names if TARGET_64BIT.
   w -- print the operand as if it's a "word" (HImode) even if it isn't.
   s -- print a shift double count, followed by the assemblers argument
	delimiter.
   b -- print the QImode name of the register for the indicated operand.
	%b0 would print %al if operands[0] is reg 0.
   w --  likewise, print the HImode name of the register.
   k --  likewise, print the SImode name of the register.
   q --  likewise, print the DImode name of the register.
   x --  likewise, print the V4SFmode name of the register.
   t --  likewise, print the V8SFmode name of the register.
   g --  likewise, print the V16SFmode name of the register.
   h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
   y -- print "st(0)" instead of "st" as a register.
   d -- print duplicated register operand for AVX instruction.
   D -- print condition for SSE cmp instruction.
   P -- if PIC, print an @PLT suffix.  For -fno-plt, load function
	address from GOT.
   p -- print raw symbol name.
   X -- don't print any sort of PIC '@' suffix for a symbol.
   & -- print some in-use local-dynamic symbol name.
   H -- print a memory address offset by 8; used for sse high-parts
   Y -- print condition for XOP pcom* instruction.
   V -- print naked full integer register name without %.
   + -- print a branch hint as 'cs' or 'ds' prefix
   ; -- print a semicolon (after prefixes due to bug in older gas).
   ~ -- print "i" if TARGET_AVX2, "f" otherwise.
   ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
   M -- print addr32 prefix for TARGET_X32 with VSIB address.
   ! -- print NOTRACK prefix for jxx/call/ret instructions if required.
   N -- print maskz if it's constant 0 operand.
 */

void
ix86_print_operand (FILE *file, rtx x, int code)
{
  if (code)
    {
      switch (code)
	{
	case 'A':
	  switch (ASSEMBLER_DIALECT)
	    {
	    case ASM_ATT:
	      putc ('*', file);
	      break;

	    case ASM_INTEL:
	      /* Intel syntax. For absolute addresses, registers should not
		 be surrounded by braces.  */
	      if (!REG_P (x))
		{
		  putc ('[', file);
		  ix86_print_operand (file, x, 0);
		  putc (']', file);
		  return;
		}
	      break;

	    default:
	      gcc_unreachable ();
	    }

	  ix86_print_operand (file, x, 0);
	  return;

	case 'E':
	  /* Wrap address in an UNSPEC to declare special handling.  */
	  if (TARGET_64BIT)
	    x = gen_rtx_UNSPEC (DImode, gen_rtvec (1, x), UNSPEC_LEA_ADDR);

	  output_address (VOIDmode, x);
	  return;

	case 'L':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('l', file);
	  return;

	case 'W':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('w', file);
	  return;

	case 'B':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('b', file);
	  return;

	case 'Q':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('l', file);
	  return;

	case 'S':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('s', file);
	  return;

	case 'T':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('t', file);
	  return;

	case 'O':
#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
	  if (ASSEMBLER_DIALECT != ASM_ATT)
	    return;

	  switch (GET_MODE_SIZE (GET_MODE (x)))
	    {
	    case 2:
	      putc ('w', file);
	      break;
  
	    case 4:
	      putc ('l', file);
	      break;

	    case 8:
	      putc ('q', file);
	      break;

	    default:
	      output_operand_lossage ("invalid operand size for operand "
				      "code 'O'");
	      return;
	    }

	  putc ('.', file);
#endif
	  return;

	case 'z':
	  if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
	    {
	      /* Opcodes don't get size suffixes if using Intel opcodes.  */
	      if (ASSEMBLER_DIALECT == ASM_INTEL)
		return;

	      switch (GET_MODE_SIZE (GET_MODE (x)))
		{
		case 1:
		  putc ('b', file);
		  return;

		case 2:
		  putc ('w', file);
		  return;

		case 4:
		  putc ('l', file);
		  return;

		case 8:
		  putc ('q', file);
		  return;

		default:
		  output_operand_lossage ("invalid operand size for operand "
					  "code 'z'");
		  return;
		}
	    }

	  if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
	    {
	      if (this_is_asm_operands)
		warning_for_asm (this_is_asm_operands,
				 "non-integer operand used with operand code %<z%>");
	      else
		warning (0, "non-integer operand used with operand code %<z%>");
	    }
	  /* FALLTHRU */

	case 'Z':
	  /* 387 opcodes don't get size suffixes if using Intel opcodes.  */
	  if (ASSEMBLER_DIALECT == ASM_INTEL)
	    return;

	  if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
	    {
	      switch (GET_MODE_SIZE (GET_MODE (x)))
		{
		case 2:
#ifdef HAVE_AS_IX86_FILDS
		  putc ('s', file);
#endif
		  return;

		case 4:
		  putc ('l', file);
		  return;

		case 8:
#ifdef HAVE_AS_IX86_FILDQ
		  putc ('q', file);
#else
		  fputs ("ll", file);
#endif
		  return;

		default:
		  break;
		}
	    }
	  else if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
	    {
	      /* 387 opcodes don't get size suffixes
		 if the operands are registers.  */
	      if (STACK_REG_P (x))
		return;

	      switch (GET_MODE_SIZE (GET_MODE (x)))
		{
		case 4:
		  putc ('s', file);
		  return;

		case 8:
		  putc ('l', file);
		  return;

		case 12:
		case 16:
		  putc ('t', file);
		  return;

		default:
		  break;
		}
	    }
	  else
	    {
	      output_operand_lossage ("invalid operand type used with "
				      "operand code '%c'", code);
	      return;
	    }

	  output_operand_lossage ("invalid operand size for operand code '%c'",
				  code);
	  return;

	case 'd':
	case 'b':
	case 'w':
	case 'k':
	case 'q':
	case 'h':
	case 't':
	case 'g':
	case 'y':
	case 'x':
	case 'X':
	case 'P':
	case 'p':
	case 'V':
	  break;

	case 's':
	  if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT)
	    {
	      ix86_print_operand (file, x, 0);
	      fputs (", ", file);
	    }
	  return;

	case 'Y':
	  switch (GET_CODE (x))
	    {
	    case NE:
	      fputs ("neq", file);
	      break;
	    case EQ:
	      fputs ("eq", file);
	      break;
	    case GE:
	    case GEU:
	      fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "ge" : "unlt", file);
	      break;
	    case GT:
	    case GTU:
	      fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "gt" : "unle", file);
	      break;
	    case LE:
	    case LEU:
	      fputs ("le", file);
	      break;
	    case LT:
	    case LTU:
	      fputs ("lt", file);
	      break;
	    case UNORDERED:
	      fputs ("unord", file);
	      break;
	    case ORDERED:
	      fputs ("ord", file);
	      break;
	    case UNEQ:
	      fputs ("ueq", file);
	      break;
	    case UNGE:
	      fputs ("nlt", file);
	      break;
	    case UNGT:
	      fputs ("nle", file);
	      break;
	    case UNLE:
	      fputs ("ule", file);
	      break;
	    case UNLT:
	      fputs ("ult", file);
	      break;
	    case LTGT:
	      fputs ("une", file);
	      break;
	    default:
	      output_operand_lossage ("operand is not a condition code, "
				      "invalid operand code 'Y'");
	      return;
	    }
	  return;

	case 'D':
	  /* Little bit of braindamage here.  The SSE compare instructions
	     does use completely different names for the comparisons that the
	     fp conditional moves.  */
	  switch (GET_CODE (x))
	    {
	    case UNEQ:
	      if (TARGET_AVX)
		{
		  fputs ("eq_us", file);
		  break;
		}
	     /* FALLTHRU */
	    case EQ:
	      fputs ("eq", file);
	      break;
	    case UNLT:
	      if (TARGET_AVX)
		{
		  fputs ("nge", file);
		  break;
		}
	     /* FALLTHRU */
	    case LT:
	      fputs ("lt", file);
	      break;
	    case UNLE:
	      if (TARGET_AVX)
		{
		  fputs ("ngt", file);
		  break;
		}
	     /* FALLTHRU */
	    case LE:
	      fputs ("le", file);
	      break;
	    case UNORDERED:
	      fputs ("unord", file);
	      break;
	    case LTGT:
	      if (TARGET_AVX)
		{
		  fputs ("neq_oq", file);
		  break;
		}
	     /* FALLTHRU */
	    case NE:
	      fputs ("neq", file);
	      break;
	    case GE:
	      if (TARGET_AVX)
		{
		  fputs ("ge", file);
		  break;
		}
	     /* FALLTHRU */
	    case UNGE:
	      fputs ("nlt", file);
	      break;
	    case GT:
	      if (TARGET_AVX)
		{
		  fputs ("gt", file);
		  break;
		}
	     /* FALLTHRU */
	    case UNGT:
	      fputs ("nle", file);
	      break;
	    case ORDERED:
	      fputs ("ord", file);
	      break;
	    default:
	      output_operand_lossage ("operand is not a condition code, "
				      "invalid operand code 'D'");
	      return;
	    }
	  return;

	case 'F':
	case 'f':
#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('.', file);
	  gcc_fallthrough ();
#endif

	case 'C':
	case 'c':
	  if (!COMPARISON_P (x))
	    {
	      output_operand_lossage ("operand is not a condition code, "
				      "invalid operand code '%c'", code);
	      return;
	    }
	  put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)),
			      code == 'c' || code == 'f',
			      code == 'F' || code == 'f',
			      file);
	  return;

	case 'H':
	  if (!offsettable_memref_p (x))
	    {
	      output_operand_lossage ("operand is not an offsettable memory "
				      "reference, invalid operand code 'H'");
	      return;
	    }
	  /* It doesn't actually matter what mode we use here, as we're
	     only going to use this for printing.  */
	  x = adjust_address_nv (x, DImode, 8);
	  /* Output 'qword ptr' for intel assembler dialect.  */
	  if (ASSEMBLER_DIALECT == ASM_INTEL)
	    code = 'q';
	  break;

	case 'K':
	  if (!CONST_INT_P (x))
	    {
	      output_operand_lossage ("operand is not an integer, invalid "
				      "operand code 'K'");
	      return;
	    }

	  if (INTVAL (x) & IX86_HLE_ACQUIRE)
#ifdef HAVE_AS_IX86_HLE
	    fputs ("xacquire ", file);
#else
	    fputs ("\n" ASM_BYTE "0xf2\n\t", file);
#endif
	  else if (INTVAL (x) & IX86_HLE_RELEASE)
#ifdef HAVE_AS_IX86_HLE
	    fputs ("xrelease ", file);
#else
	    fputs ("\n" ASM_BYTE "0xf3\n\t", file);
#endif
	  /* We do not want to print value of the operand.  */
	  return;

	case 'N':
	  if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
	    fputs ("{z}", file);
	  return;

	case 'r':
	  if (!CONST_INT_P (x) || INTVAL (x) != ROUND_SAE)
	    {
	      output_operand_lossage ("operand is not a specific integer, "
				      "invalid operand code 'r'");
	      return;
	    }

	  if (ASSEMBLER_DIALECT == ASM_INTEL)
	    fputs (", ", file);

	  fputs ("{sae}", file);

	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    fputs (", ", file);

	  return;

	case 'R':
	  if (!CONST_INT_P (x))
	    {
	      output_operand_lossage ("operand is not an integer, invalid "
				      "operand code 'R'");
	      return;
	    }

	  if (ASSEMBLER_DIALECT == ASM_INTEL)
	    fputs (", ", file);

	  switch (INTVAL (x))
	    {
	    case ROUND_NEAREST_INT | ROUND_SAE:
	      fputs ("{rn-sae}", file);
	      break;
	    case ROUND_NEG_INF | ROUND_SAE:
	      fputs ("{rd-sae}", file);
	      break;
	    case ROUND_POS_INF | ROUND_SAE:
	      fputs ("{ru-sae}", file);
	      break;
	    case ROUND_ZERO | ROUND_SAE:
	      fputs ("{rz-sae}", file);
	      break;
	    default:
	      output_operand_lossage ("operand is not a specific integer, "
				      "invalid operand code 'R'");
	    }

	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    fputs (", ", file);

	  return;

	case '*':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('*', file);
	  return;

	case '&':
	  {
	    const char *name = get_some_local_dynamic_name ();
	    if (name == NULL)
	      output_operand_lossage ("'%%&' used without any "
				      "local dynamic TLS references");
	    else
	      assemble_name (file, name);
	    return;
	  }

	case '+':
	  {
	    rtx x;

	    if (!optimize
	        || optimize_function_for_size_p (cfun)
		|| !TARGET_BRANCH_PREDICTION_HINTS)
	      return;

	    x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
	    if (x)
	      {
		int pred_val = profile_probability::from_reg_br_prob_note
				 (XINT (x, 0)).to_reg_br_prob_base ();

		if (pred_val < REG_BR_PROB_BASE * 45 / 100
		    || pred_val > REG_BR_PROB_BASE * 55 / 100)
		  {
		    bool taken = pred_val > REG_BR_PROB_BASE / 2;
		    bool cputaken
		      = final_forward_branch_p (current_output_insn) == 0;

		    /* Emit hints only in the case default branch prediction
		       heuristics would fail.  */
		    if (taken != cputaken)
		      {
			/* We use 3e (DS) prefix for taken branches and
			   2e (CS) prefix for not taken branches.  */
			if (taken)
			  fputs ("ds ; ", file);
			else
			  fputs ("cs ; ", file);
		      }
		  }
	      }
	    return;
	  }

	case ';':
#ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
	  putc (';', file);
#endif
	  return;

	case '~':
	  putc (TARGET_AVX2 ? 'i' : 'f', file);
	  return;

	case 'M':
	  if (TARGET_X32)
	    {
	      /* NB: 32-bit indices in VSIB address are sign-extended
		 to 64 bits. In x32, if 32-bit address 0xf7fa3010 is
		 sign-extended to 0xfffffffff7fa3010 which is invalid
		 address.  Add addr32 prefix if there is no base
		 register nor symbol.  */
	      bool ok;
	      struct ix86_address parts;
	      ok = ix86_decompose_address (x, &parts);
	      gcc_assert (ok && parts.index == NULL_RTX);
	      if (parts.base == NULL_RTX
		  && (parts.disp == NULL_RTX
		      || !symbolic_operand (parts.disp,
					    GET_MODE (parts.disp))))
		fputs ("addr32 ", file);
	    }
	  return;

	case '^':
	  if (TARGET_64BIT && Pmode != word_mode)
	    fputs ("addr32 ", file);
	  return;

	case '!':
	  if (ix86_notrack_prefixed_insn_p (current_output_insn))
	    fputs ("notrack ", file);
	  return;

	default:
	  output_operand_lossage ("invalid operand code '%c'", code);
	}
    }

  if (REG_P (x))
    print_reg (x, code, file);

  else if (MEM_P (x))
    {
      rtx addr = XEXP (x, 0);

      /* No `byte ptr' prefix for call instructions ... */
      if (ASSEMBLER_DIALECT == ASM_INTEL && code != 'X' && code != 'P')
	{
	  machine_mode mode = GET_MODE (x);
	  const char *size;

	  /* Check for explicit size override codes.  */
	  if (code == 'b')
	    size = "BYTE";
	  else if (code == 'w')
	    size = "WORD";
	  else if (code == 'k')
	    size = "DWORD";
	  else if (code == 'q')
	    size = "QWORD";
	  else if (code == 'x')
	    size = "XMMWORD";
	  else if (code == 't')
	    size = "YMMWORD";
	  else if (code == 'g')
	    size = "ZMMWORD";
	  else if (mode == BLKmode)
	    /* ... or BLKmode operands, when not overridden.  */
	    size = NULL;
	  else
	    switch (GET_MODE_SIZE (mode))
	      {
	      case 1: size = "BYTE"; break;
	      case 2: size = "WORD"; break;
	      case 4: size = "DWORD"; break;
	      case 8: size = "QWORD"; break;
	      case 12: size = "TBYTE"; break;
	      case 16:
		if (mode == XFmode)
		  size = "TBYTE";
		else
		  size = "XMMWORD";
		break;
	      case 32: size = "YMMWORD"; break;
	      case 64: size = "ZMMWORD"; break;
	      default:
		gcc_unreachable ();
	      }
	  if (size)
	    {
	      fputs (size, file);
	      fputs (" PTR ", file);
	    }
	}

      if (this_is_asm_operands && ! address_operand (addr, VOIDmode))
	output_operand_lossage ("invalid constraints for operand");
      else
	ix86_print_operand_address_as
	  (file, addr, MEM_ADDR_SPACE (x), code == 'p' || code == 'P');
    }

  else if (CONST_DOUBLE_P (x) && GET_MODE (x) == HFmode)
    {
      long l = real_to_target (NULL, CONST_DOUBLE_REAL_VALUE (x),
			       REAL_MODE_FORMAT (HFmode));
      if (ASSEMBLER_DIALECT == ASM_ATT)
	putc ('$', file);
      fprintf (file, "0x%04x", (unsigned int) l);
    }

  else if (CONST_DOUBLE_P (x) && GET_MODE (x) == SFmode)
    {
      long l;

      REAL_VALUE_TO_TARGET_SINGLE (*CONST_DOUBLE_REAL_VALUE (x), l);

      if (ASSEMBLER_DIALECT == ASM_ATT)
	putc ('$', file);
      /* Sign extend 32bit SFmode immediate to 8 bytes.  */
      if (code == 'q')
	fprintf (file, "0x%08" HOST_LONG_LONG_FORMAT "x",
		 (unsigned long long) (int) l);
      else
	fprintf (file, "0x%08x", (unsigned int) l);
    }

  else if (CONST_DOUBLE_P (x) && GET_MODE (x) == DFmode)
    {
      long l[2];

      REAL_VALUE_TO_TARGET_DOUBLE (*CONST_DOUBLE_REAL_VALUE (x), l);

      if (ASSEMBLER_DIALECT == ASM_ATT)
	putc ('$', file);
      fprintf (file, "0x%lx%08lx", l[1] & 0xffffffff, l[0] & 0xffffffff);
    }

  /* These float cases don't actually occur as immediate operands.  */
  else if (CONST_DOUBLE_P (x) && GET_MODE (x) == XFmode)
    {
      char dstr[30];

      real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
      fputs (dstr, file);
    }

  /* Print bcst_mem_operand.  */
  else if (GET_CODE (x) == VEC_DUPLICATE)
    {
      machine_mode vmode = GET_MODE (x);
      /* Must be bcst_memory_operand.  */
      gcc_assert (bcst_mem_operand (x, vmode));

      rtx mem = XEXP (x,0);
      ix86_print_operand (file, mem, 0);

      switch (vmode)
	{
	case E_V2DImode:
	case E_V2DFmode:
	  fputs ("{1to2}", file);
	  break;
	case E_V4SImode:
	case E_V4SFmode:
	case E_V4DImode:
	case E_V4DFmode:
	  fputs ("{1to4}", file);
	  break;
	case E_V8SImode:
	case E_V8SFmode:
	case E_V8DFmode:
	case E_V8DImode:
	case E_V8HFmode:
	  fputs ("{1to8}", file);
	  break;
	case E_V16SFmode:
	case E_V16SImode:
	case E_V16HFmode:
	  fputs ("{1to16}", file);
	  break;
	case E_V32HFmode:
	  fputs ("{1to32}", file);
	  break;
	default:
	  gcc_unreachable ();
	}
    }

  else
    {
      /* We have patterns that allow zero sets of memory, for instance.
	 In 64-bit mode, we should probably support all 8-byte vectors,
	 since we can in fact encode that into an immediate.  */
      if (GET_CODE (x) == CONST_VECTOR)
	{
	  if (x != CONST0_RTX (GET_MODE (x)))
	    output_operand_lossage ("invalid vector immediate");
	  x = const0_rtx;
	}

      if (code == 'P')
	{
	  if (ix86_force_load_from_GOT_p (x, true))
	    {
	      /* For inline assembly statement, load function address
		 from GOT with 'P' operand modifier to avoid PLT.  */
	      x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
				  (TARGET_64BIT
				   ? UNSPEC_GOTPCREL
				   : UNSPEC_GOT));
	      x = gen_rtx_CONST (Pmode, x);
	      x = gen_const_mem (Pmode, x);
	      ix86_print_operand (file, x, 'A');
	      return;
	    }
	}
      else if (code != 'p')
	{
	  if (CONST_INT_P (x))
	    {
	      if (ASSEMBLER_DIALECT == ASM_ATT)
		putc ('$', file);
	    }
	  else if (GET_CODE (x) == CONST || GET_CODE (x) == SYMBOL_REF
		   || GET_CODE (x) == LABEL_REF)
	    {
	      if (ASSEMBLER_DIALECT == ASM_ATT)
		putc ('$', file);
	      else
		fputs ("OFFSET FLAT:", file);
	    }
	}
      if (CONST_INT_P (x))
	fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
      else if (flag_pic || MACHOPIC_INDIRECT)
	output_pic_addr_const (file, x, code);
      else
	output_addr_const (file, x);
    }
}

static bool
ix86_print_operand_punct_valid_p (unsigned char code)
{
  return (code == '*' || code == '+' || code == '&' || code == ';'
	  || code == '~' || code == '^' || code == '!');
}

/* Print a memory operand whose address is ADDR.  */

static void
ix86_print_operand_address_as (FILE *file, rtx addr,
			       addr_space_t as, bool raw)
{
  struct ix86_address parts;
  rtx base, index, disp;
  int scale;
  int ok;
  bool vsib = false;
  int code = 0;

  if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_VSIBADDR)
    {
      ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
      gcc_assert (parts.index == NULL_RTX);
      parts.index = XVECEXP (addr, 0, 1);
      parts.scale = INTVAL (XVECEXP (addr, 0, 2));
      addr = XVECEXP (addr, 0, 0);
      vsib = true;
    }
  else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_LEA_ADDR)
    {
      gcc_assert (TARGET_64BIT);
      ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
      code = 'q';
    }
  else
    ok = ix86_decompose_address (addr, &parts);

  gcc_assert (ok);

  base = parts.base;
  index = parts.index;
  disp = parts.disp;
  scale = parts.scale;

  if (ADDR_SPACE_GENERIC_P (as))
    as = parts.seg;
  else
    gcc_assert (ADDR_SPACE_GENERIC_P (parts.seg));

  if (!ADDR_SPACE_GENERIC_P (as) && !raw)
    {
      if (ASSEMBLER_DIALECT == ASM_ATT)
	putc ('%', file);

      switch (as)
	{
	case ADDR_SPACE_SEG_FS:
	  fputs ("fs:", file);
	  break;
	case ADDR_SPACE_SEG_GS:
	  fputs ("gs:", file);
	  break;
	default:
	  gcc_unreachable ();
	}
    }

  /* Use one byte shorter RIP relative addressing for 64bit mode.  */
  if (TARGET_64BIT && !base && !index && !raw)
    {
      rtx symbol = disp;

      if (GET_CODE (disp) == CONST
	  && GET_CODE (XEXP (disp, 0)) == PLUS
	  && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
	symbol = XEXP (XEXP (disp, 0), 0);

      if (GET_CODE (symbol) == LABEL_REF
	  || (GET_CODE (symbol) == SYMBOL_REF
	      && SYMBOL_REF_TLS_MODEL (symbol) == 0))
	base = pc_rtx;
    }

  if (!base && !index)
    {
      /* Displacement only requires special attention.  */
      if (CONST_INT_P (disp))
	{
	  if (ASSEMBLER_DIALECT == ASM_INTEL && ADDR_SPACE_GENERIC_P (as))
	    fputs ("ds:", file);
	  fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (disp));
	}
      /* Load the external function address via the GOT slot to avoid PLT.  */
      else if (GET_CODE (disp) == CONST
	       && GET_CODE (XEXP (disp, 0)) == UNSPEC
	       && (XINT (XEXP (disp, 0), 1) == UNSPEC_GOTPCREL
		   || XINT (XEXP (disp, 0), 1) == UNSPEC_GOT)
	       && ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
	output_pic_addr_const (file, disp, 0);
      else if (flag_pic)
	output_pic_addr_const (file, disp, 0);
      else
	output_addr_const (file, disp);
    }
  else
    {
      /* Print SImode register names to force addr32 prefix.  */
      if (SImode_address_operand (addr, VOIDmode))
	{
	  if (flag_checking)
	    {
	      gcc_assert (TARGET_64BIT);
	      switch (GET_CODE (addr))
		{
		case SUBREG:
		  gcc_assert (GET_MODE (addr) == SImode);
		  gcc_assert (GET_MODE (SUBREG_REG (addr)) == DImode);
		  break;
		case ZERO_EXTEND:
		case AND:
		  gcc_assert (GET_MODE (addr) == DImode);
		  break;
		default:
		  gcc_unreachable ();
		}
	    }
	  gcc_assert (!code);
	  code = 'k';
	}
      else if (code == 0
	       && TARGET_X32
	       && disp
	       && CONST_INT_P (disp)
	       && INTVAL (disp) < -16*1024*1024)
	{
	  /* X32 runs in 64-bit mode, where displacement, DISP, in
	     address DISP(%r64), is encoded as 32-bit immediate sign-
	     extended from 32-bit to 64-bit.  For -0x40000300(%r64),
	     address is %r64 + 0xffffffffbffffd00.  When %r64 <
	     0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
	     which is invalid for x32.  The correct address is %r64
	     - 0x40000300 == 0xf7ffdd64.  To properly encode
	     -0x40000300(%r64) for x32, we zero-extend negative
	     displacement by forcing addr32 prefix which truncates
	     0xfffffffff7ffdd64 to 0xf7ffdd64.  In theory, we should
	     zero-extend all negative displacements, including -1(%rsp).
	     However, for small negative displacements, sign-extension
	     won't cause overflow.  We only zero-extend negative
	     displacements if they < -16*1024*1024, which is also used
	     to check legitimate address displacements for PIC.  */
	  code = 'k';
	}

      /* Since the upper 32 bits of RSP are always zero for x32,
	 we can encode %esp as %rsp to avoid 0x67 prefix if
	 there is no index register.  */
      if (TARGET_X32 && Pmode == SImode
	  && !index && base && REG_P (base) && REGNO (base) == SP_REG)
	code = 'q';

      if (ASSEMBLER_DIALECT == ASM_ATT)
	{
	  if (disp)
	    {
	      if (flag_pic)
		output_pic_addr_const (file, disp, 0);
	      else if (GET_CODE (disp) == LABEL_REF)
		output_asm_label (disp);
	      else
		output_addr_const (file, disp);
	    }

	  putc ('(', file);
	  if (base)
	    print_reg (base, code, file);
	  if (index)
	    {
	      putc (',', file);
	      print_reg (index, vsib ? 0 : code, file);
	      if (scale != 1 || vsib)
		fprintf (file, ",%d", scale);
	    }
	  putc (')', file);
	}
      else
	{
	  rtx offset = NULL_RTX;

	  if (disp)
	    {
	      /* Pull out the offset of a symbol; print any symbol itself.  */
	      if (GET_CODE (disp) == CONST
		  && GET_CODE (XEXP (disp, 0)) == PLUS
		  && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
		{
		  offset = XEXP (XEXP (disp, 0), 1);
		  disp = gen_rtx_CONST (VOIDmode,
					XEXP (XEXP (disp, 0), 0));
		}

	      if (flag_pic)
		output_pic_addr_const (file, disp, 0);
	      else if (GET_CODE (disp) == LABEL_REF)
		output_asm_label (disp);
	      else if (CONST_INT_P (disp))
		offset = disp;
	      else
		output_addr_const (file, disp);
	    }

	  putc ('[', file);
	  if (base)
	    {
	      print_reg (base, code, file);
	      if (offset)
		{
		  if (INTVAL (offset) >= 0)
		    putc ('+', file);
		  fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
		}
	    }
	  else if (offset)
	    fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
	  else
	    putc ('0', file);

	  if (index)
	    {
	      putc ('+', file);
	      print_reg (index, vsib ? 0 : code, file);
	      if (scale != 1 || vsib)
		fprintf (file, "*%d", scale);
	    }
	  putc (']', file);
	}
    }
}

static void
ix86_print_operand_address (FILE *file, machine_mode /*mode*/, rtx addr)
{
  if (this_is_asm_operands && ! address_operand (addr, VOIDmode))
    output_operand_lossage ("invalid constraints for operand");
  else
    ix86_print_operand_address_as (file, addr, ADDR_SPACE_GENERIC, false);
}

/* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA.  */

static bool
i386_asm_output_addr_const_extra (FILE *file, rtx x)
{
  rtx op;

  if (GET_CODE (x) != UNSPEC)
    return false;

  op = XVECEXP (x, 0, 0);
  switch (XINT (x, 1))
    {
    case UNSPEC_GOTOFF:
      output_addr_const (file, op);
      fputs ("@gotoff", file);
      break;
    case UNSPEC_GOTTPOFF:
      output_addr_const (file, op);
      /* FIXME: This might be @TPOFF in Sun ld.  */
      fputs ("@gottpoff", file);
      break;
    case UNSPEC_TPOFF:
      output_addr_const (file, op);
      fputs ("@tpoff", file);
      break;
    case UNSPEC_NTPOFF:
      output_addr_const (file, op);
      if (TARGET_64BIT)
	fputs ("@tpoff", file);
      else
	fputs ("@ntpoff", file);
      break;
    case UNSPEC_DTPOFF:
      output_addr_const (file, op);
      fputs ("@dtpoff", file);
      break;
    case UNSPEC_GOTNTPOFF:
      output_addr_const (file, op);
      if (TARGET_64BIT)
	fputs (ASSEMBLER_DIALECT == ASM_ATT ?
	       "@gottpoff(%rip)" : "@gottpoff[rip]", file);
      else
	fputs ("@gotntpoff", file);
      break;
    case UNSPEC_INDNTPOFF:
      output_addr_const (file, op);
      fputs ("@indntpoff", file);
      break;
#if TARGET_MACHO
    case UNSPEC_MACHOPIC_OFFSET:
      output_addr_const (file, op);
      putc ('-', file);
      machopic_output_function_base_name (file);
      break;
#endif

    default:
      return false;
    }

  return true;
}


/* Output code to perform a 387 binary operation in INSN, one of PLUS,
   MINUS, MULT or DIV.  OPERANDS are the insn operands, where operands[3]
   is the expression of the binary operation.  The output may either be
   emitted here, or returned to the caller, like all output_* functions.

   There is no guarantee that the operands are the same mode, as they
   might be within FLOAT or FLOAT_EXTEND expressions.  */

#ifndef SYSV386_COMPAT
/* Set to 1 for compatibility with brain-damaged assemblers.  No-one
   wants to fix the assemblers because that causes incompatibility
   with gcc.  No-one wants to fix gcc because that causes
   incompatibility with assemblers...  You can use the option of
   -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way.  */
#define SYSV386_COMPAT 1
#endif

const char *
output_387_binary_op (rtx_insn *insn, rtx *operands)
{
  static char buf[40];
  const char *p;
  bool is_sse
    = (SSE_REG_P (operands[0])
       || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]));

  if (is_sse)
    p = "%v";
  else if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
	   || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
    p = "fi";
  else
    p = "f";

  strcpy (buf, p);

  switch (GET_CODE (operands[3]))
    {
    case PLUS:
      p = "add"; break;
    case MINUS:
      p = "sub"; break;
    case MULT:
      p = "mul"; break;
    case DIV:
      p = "div"; break;
    default:
      gcc_unreachable ();
    }

  strcat (buf, p);

  if (is_sse)
   {
     p = GET_MODE (operands[0]) == SFmode ? "ss" : "sd";
     strcat (buf, p);

     if (TARGET_AVX)
       p = "\t{%2, %1, %0|%0, %1, %2}";
     else
       p = "\t{%2, %0|%0, %2}";

     strcat (buf, p);
     return buf;
   }

  /* Even if we do not want to check the inputs, this documents input
     constraints.  Which helps in understanding the following code.  */
  if (flag_checking)
    {
      if (STACK_REG_P (operands[0])
	  && ((REG_P (operands[1])
	       && REGNO (operands[0]) == REGNO (operands[1])
	       && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
	      || (REG_P (operands[2])
		  && REGNO (operands[0]) == REGNO (operands[2])
		  && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
	  && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
	; /* ok */
      else
	gcc_unreachable ();
    }

  switch (GET_CODE (operands[3]))
    {
    case MULT:
    case PLUS:
      if (REG_P (operands[2]) && REGNO (operands[0]) == REGNO (operands[2]))
	std::swap (operands[1], operands[2]);

      /* know operands[0] == operands[1].  */

      if (MEM_P (operands[2]))
	{
	  p = "%Z2\t%2";
	  break;
	}

      if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
	{
	  if (STACK_TOP_P (operands[0]))
	    /* How is it that we are storing to a dead operand[2]?
	       Well, presumably operands[1] is dead too.  We can't
	       store the result to st(0) as st(0) gets popped on this
	       instruction.  Instead store to operands[2] (which I
	       think has to be st(1)).  st(1) will be popped later.
	       gcc <= 2.8.1 didn't have this check and generated
	       assembly code that the Unixware assembler rejected.  */
	    p = "p\t{%0, %2|%2, %0}";	/* st(1) = st(0) op st(1); pop */
	  else
	    p = "p\t{%2, %0|%0, %2}";	/* st(r1) = st(r1) op st(0); pop */
	  break;
	}

      if (STACK_TOP_P (operands[0]))
	p = "\t{%y2, %0|%0, %y2}";	/* st(0) = st(0) op st(r2) */
      else
	p = "\t{%2, %0|%0, %2}";	/* st(r1) = st(r1) op st(0) */
      break;

    case MINUS:
    case DIV:
      if (MEM_P (operands[1]))
	{
	  p = "r%Z1\t%1";
	  break;
	}

      if (MEM_P (operands[2]))
	{
	  p = "%Z2\t%2";
	  break;
	}

      if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
	{
#if SYSV386_COMPAT
	  /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
	     derived assemblers, confusingly reverse the direction of
	     the operation for fsub{r} and fdiv{r} when the
	     destination register is not st(0).  The Intel assembler
	     doesn't have this brain damage.  Read !SYSV386_COMPAT to
	     figure out what the hardware really does.  */
	  if (STACK_TOP_P (operands[0]))
	    p = "{p\t%0, %2|rp\t%2, %0}";
	  else
	    p = "{rp\t%2, %0|p\t%0, %2}";
#else
	  if (STACK_TOP_P (operands[0]))
	    /* As above for fmul/fadd, we can't store to st(0).  */
	    p = "rp\t{%0, %2|%2, %0}";	/* st(1) = st(0) op st(1); pop */
	  else
	    p = "p\t{%2, %0|%0, %2}";	/* st(r1) = st(r1) op st(0); pop */
#endif
	  break;
	}

      if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
	{
#if SYSV386_COMPAT
	  if (STACK_TOP_P (operands[0]))
	    p = "{rp\t%0, %1|p\t%1, %0}";
	  else
	    p = "{p\t%1, %0|rp\t%0, %1}";
#else
	  if (STACK_TOP_P (operands[0]))
	    p = "p\t{%0, %1|%1, %0}";	/* st(1) = st(1) op st(0); pop */
	  else
	    p = "rp\t{%1, %0|%0, %1}";	/* st(r2) = st(0) op st(r2); pop */
#endif
	  break;
	}

      if (STACK_TOP_P (operands[0]))
	{
	  if (STACK_TOP_P (operands[1]))
	    p = "\t{%y2, %0|%0, %y2}";	/* st(0) = st(0) op st(r2) */
	  else
	    p = "r\t{%y1, %0|%0, %y1}";	/* st(0) = st(r1) op st(0) */
	  break;
	}
      else if (STACK_TOP_P (operands[1]))
	{
#if SYSV386_COMPAT
	  p = "{\t%1, %0|r\t%0, %1}";
#else
	  p = "r\t{%1, %0|%0, %1}";	/* st(r2) = st(0) op st(r2) */
#endif
	}
      else
	{
#if SYSV386_COMPAT
	  p = "{r\t%2, %0|\t%0, %2}";
#else
	  p = "\t{%2, %0|%0, %2}";	/* st(r1) = st(r1) op st(0) */
#endif
	}
      break;

    default:
      gcc_unreachable ();
    }

  strcat (buf, p);
  return buf;
}

/* Return needed mode for entity in optimize_mode_switching pass.  */

static int
ix86_dirflag_mode_needed (rtx_insn *insn)
{
  if (CALL_P (insn))
    {
      if (cfun->machine->func_type == TYPE_NORMAL)
	return X86_DIRFLAG_ANY;
      else
	/* No need to emit CLD in interrupt handler for TARGET_CLD.  */
	return TARGET_CLD ? X86_DIRFLAG_ANY : X86_DIRFLAG_RESET;
    }

  if (recog_memoized (insn) < 0)
    return X86_DIRFLAG_ANY;

  if (get_attr_type (insn) == TYPE_STR)
    {
      /* Emit cld instruction if stringops are used in the function.  */
      if (cfun->machine->func_type == TYPE_NORMAL)
	return TARGET_CLD ? X86_DIRFLAG_RESET : X86_DIRFLAG_ANY;
      else
	return X86_DIRFLAG_RESET;
    }

  return X86_DIRFLAG_ANY;
}

/* Check if a 256bit or 512 bit AVX register is referenced inside of EXP.   */

static bool
ix86_check_avx_upper_register (const_rtx exp)
{
  return (SSE_REG_P (exp)
	  && !EXT_REX_SSE_REG_P (exp)
	  && GET_MODE_BITSIZE (GET_MODE (exp)) > 128);
}

/* Check if a 256bit or 512bit AVX register is referenced in stores.   */

static void
ix86_check_avx_upper_stores (rtx dest, const_rtx, void *data)
{
  if (ix86_check_avx_upper_register (dest))
    {
      bool *used = (bool *) data;
      *used = true;
    }
}

/* Return needed mode for entity in optimize_mode_switching pass.  */

static int
ix86_avx_u128_mode_needed (rtx_insn *insn)
{
  if (DEBUG_INSN_P (insn))
    return AVX_U128_ANY;

  if (CALL_P (insn))
    {
      rtx link;

      /* Needed mode is set to AVX_U128_CLEAN if there are
	 no 256bit or 512bit modes used in function arguments. */
      for (link = CALL_INSN_FUNCTION_USAGE (insn);
	   link;
	   link = XEXP (link, 1))
	{
	  if (GET_CODE (XEXP (link, 0)) == USE)
	    {
	      rtx arg = XEXP (XEXP (link, 0), 0);

	      if (ix86_check_avx_upper_register (arg))
		return AVX_U128_DIRTY;
	    }
	}

      /* Needed mode is set to AVX_U128_CLEAN if there are no 256bit
	 nor 512bit registers used in the function return register.  */
      bool avx_upper_reg_found = false;
      note_stores (insn, ix86_check_avx_upper_stores,
		   &avx_upper_reg_found);
      if (avx_upper_reg_found)
	return AVX_U128_DIRTY;

      /* If the function is known to preserve some SSE registers,
	 RA and previous passes can legitimately rely on that for
	 modes wider than 256 bits.  It's only safe to issue a
	 vzeroupper if all SSE registers are clobbered.  */
      const function_abi &abi = insn_callee_abi (insn);
      if (vzeroupper_pattern (PATTERN (insn), VOIDmode)
	  || !hard_reg_set_subset_p (reg_class_contents[SSE_REGS],
				     abi.mode_clobbers (V4DImode)))
	return AVX_U128_ANY;

      return AVX_U128_CLEAN;
    }

  subrtx_iterator::array_type array;

  rtx set = single_set (insn);
  if (set)
    {
      rtx dest = SET_DEST (set);
      rtx src = SET_SRC (set);
      if (ix86_check_avx_upper_register (dest))
	{
	  /* This is an YMM/ZMM load.  Return AVX_U128_DIRTY if the
	     source isn't zero.  */
	  if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
	    return AVX_U128_DIRTY;
	  else
	    return AVX_U128_ANY;
	}
      else
	{
	  FOR_EACH_SUBRTX (iter, array, src, NONCONST)
	    if (ix86_check_avx_upper_register (*iter))
	      return AVX_U128_DIRTY;
	}

      /* This isn't YMM/ZMM load/store.  */
      return AVX_U128_ANY;
    }

  /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
     Hardware changes state only when a 256bit register is written to,
     but we need to prevent the compiler from moving optimal insertion
     point above eventual read from 256bit or 512 bit register.  */
  FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
    if (ix86_check_avx_upper_register (*iter))
      return AVX_U128_DIRTY;

  return AVX_U128_ANY;
}

/* Return mode that i387 must be switched into
   prior to the execution of insn.  */

static int
ix86_i387_mode_needed (int entity, rtx_insn *insn)
{
  enum attr_i387_cw mode;

  /* The mode UNINITIALIZED is used to store control word after a
     function call or ASM pattern.  The mode ANY specify that function
     has no requirements on the control word and make no changes in the
     bits we are interested in.  */

  if (CALL_P (insn)
      || (NONJUMP_INSN_P (insn)
	  && (asm_noperands (PATTERN (insn)) >= 0
	      || GET_CODE (PATTERN (insn)) == ASM_INPUT)))
    return I387_CW_UNINITIALIZED;

  if (recog_memoized (insn) < 0)
    return I387_CW_ANY;

  mode = get_attr_i387_cw (insn);

  switch (entity)
    {
    case I387_ROUNDEVEN:
      if (mode == I387_CW_ROUNDEVEN)
	return mode;
      break;

    case I387_TRUNC:
      if (mode == I387_CW_TRUNC)
	return mode;
      break;

    case I387_FLOOR:
      if (mode == I387_CW_FLOOR)
	return mode;
      break;

    case I387_CEIL:
      if (mode == I387_CW_CEIL)
	return mode;
      break;

    default:
      gcc_unreachable ();
    }

  return I387_CW_ANY;
}

/* Return mode that entity must be switched into
   prior to the execution of insn.  */

static int
ix86_mode_needed (int entity, rtx_insn *insn)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      return ix86_dirflag_mode_needed (insn);
    case AVX_U128:
      return ix86_avx_u128_mode_needed (insn);
    case I387_ROUNDEVEN:
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
      return ix86_i387_mode_needed (entity, insn);
    default:
      gcc_unreachable ();
    }
  return 0;
}

/* Calculate mode of upper 128bit AVX registers after the insn.  */

static int
ix86_avx_u128_mode_after (int mode, rtx_insn *insn)
{
  rtx pat = PATTERN (insn);

  if (vzeroupper_pattern (pat, VOIDmode)
      || vzeroall_pattern (pat, VOIDmode))
    return AVX_U128_CLEAN;

  /* We know that state is clean after CALL insn if there are no
     256bit or 512bit registers used in the function return register. */
  if (CALL_P (insn))
    {
      bool avx_upper_reg_found = false;
      note_stores (insn, ix86_check_avx_upper_stores, &avx_upper_reg_found);

      return avx_upper_reg_found ? AVX_U128_DIRTY : AVX_U128_CLEAN;
    }

  /* Otherwise, return current mode.  Remember that if insn
     references AVX 256bit or 512bit registers, the mode was already
     changed to DIRTY from MODE_NEEDED.  */
  return mode;
}

/* Return the mode that an insn results in.  */

static int
ix86_mode_after (int entity, int mode, rtx_insn *insn)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      return mode;
    case AVX_U128:
      return ix86_avx_u128_mode_after (mode, insn);
    case I387_ROUNDEVEN:
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
      return mode;
    default:
      gcc_unreachable ();
    }
}

static int
ix86_dirflag_mode_entry (void)
{
  /* For TARGET_CLD or in the interrupt handler we can't assume
     direction flag state at function entry.  */
  if (TARGET_CLD
      || cfun->machine->func_type != TYPE_NORMAL)
    return X86_DIRFLAG_ANY;

  return X86_DIRFLAG_RESET;
}

static int
ix86_avx_u128_mode_entry (void)
{
  tree arg;

  /* Entry mode is set to AVX_U128_DIRTY if there are
     256bit or 512bit modes used in function arguments.  */
  for (arg = DECL_ARGUMENTS (current_function_decl); arg;
       arg = TREE_CHAIN (arg))
    {
      rtx incoming = DECL_INCOMING_RTL (arg);

      if (incoming && ix86_check_avx_upper_register (incoming))
	return AVX_U128_DIRTY;
    }

  return AVX_U128_CLEAN;
}

/* Return a mode that ENTITY is assumed to be
   switched to at function entry.  */

static int
ix86_mode_entry (int entity)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      return ix86_dirflag_mode_entry ();
    case AVX_U128:
      return ix86_avx_u128_mode_entry ();
    case I387_ROUNDEVEN:
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
      return I387_CW_ANY;
    default:
      gcc_unreachable ();
    }
}

static int
ix86_avx_u128_mode_exit (void)
{
  rtx reg = crtl->return_rtx;

  /* Exit mode is set to AVX_U128_DIRTY if there are 256bit
     or 512 bit modes used in the function return register. */
  if (reg && ix86_check_avx_upper_register (reg))
    return AVX_U128_DIRTY;

  /* Exit mode is set to AVX_U128_DIRTY if there are 256bit or 512bit
     modes used in function arguments, otherwise return AVX_U128_CLEAN.
   */
  return ix86_avx_u128_mode_entry ();
}

/* Return a mode that ENTITY is assumed to be
   switched to at function exit.  */

static int
ix86_mode_exit (int entity)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      return X86_DIRFLAG_ANY;
    case AVX_U128:
      return ix86_avx_u128_mode_exit ();
    case I387_ROUNDEVEN:
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
      return I387_CW_ANY;
    default:
      gcc_unreachable ();
    }
}

static int
ix86_mode_priority (int, int n)
{
  return n;
}

/* Output code to initialize control word copies used by trunc?f?i and
   rounding patterns.  CURRENT_MODE is set to current control word,
   while NEW_MODE is set to new control word.  */

static void
emit_i387_cw_initialization (int mode)
{
  rtx stored_mode = assign_386_stack_local (HImode, SLOT_CW_STORED);
  rtx new_mode;

  enum ix86_stack_slot slot;

  rtx reg = gen_reg_rtx (HImode);

  emit_insn (gen_x86_fnstcw_1 (stored_mode));
  emit_move_insn (reg, copy_rtx (stored_mode));

  switch (mode)
    {
    case I387_CW_ROUNDEVEN:
      /* round to nearest */
      emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
      slot = SLOT_CW_ROUNDEVEN;
      break;

    case I387_CW_TRUNC:
      /* round toward zero (truncate) */
      emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00)));
      slot = SLOT_CW_TRUNC;
      break;

    case I387_CW_FLOOR:
      /* round down toward -oo */
      emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
      emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400)));
      slot = SLOT_CW_FLOOR;
      break;

    case I387_CW_CEIL:
      /* round up toward +oo */
      emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
      emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800)));
      slot = SLOT_CW_CEIL;
      break;

    default:
      gcc_unreachable ();
    }

  gcc_assert (slot < MAX_386_STACK_LOCALS);

  new_mode = assign_386_stack_local (HImode, slot);
  emit_move_insn (new_mode, reg);
}

/* Generate one or more insns to set ENTITY to MODE.  */

static void
ix86_emit_mode_set (int entity, int mode, int prev_mode ATTRIBUTE_UNUSED,
		    HARD_REG_SET regs_live ATTRIBUTE_UNUSED)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      if (mode == X86_DIRFLAG_RESET)
	emit_insn (gen_cld ());
      break;
    case AVX_U128:
      if (mode == AVX_U128_CLEAN)
	ix86_expand_avx_vzeroupper ();
      break;
    case I387_ROUNDEVEN:
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
      if (mode != I387_CW_ANY
	  && mode != I387_CW_UNINITIALIZED)
	emit_i387_cw_initialization (mode);
      break;
    default:
      gcc_unreachable ();
    }
}

/* Output code for INSN to convert a float to a signed int.  OPERANDS
   are the insn operands.  The output may be [HSD]Imode and the input
   operand may be [SDX]Fmode.  */

const char *
output_fix_trunc (rtx_insn *insn, rtx *operands, bool fisttp)
{
  bool stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG);
  bool dimode_p = GET_MODE (operands[0]) == DImode;
  int round_mode = get_attr_i387_cw (insn);

  static char buf[40];
  const char *p;

  /* Jump through a hoop or two for DImode, since the hardware has no
     non-popping instruction.  We used to do this a different way, but
     that was somewhat fragile and broke with post-reload splitters.  */
  if ((dimode_p || fisttp) && !stack_top_dies)
    output_asm_insn ("fld\t%y1", operands);

  gcc_assert (STACK_TOP_P (operands[1]));
  gcc_assert (MEM_P (operands[0]));
  gcc_assert (GET_MODE (operands[1]) != TFmode);

  if (fisttp)
    return "fisttp%Z0\t%0";

  strcpy (buf, "fist");

  if (round_mode != I387_CW_ANY)
    output_asm_insn ("fldcw\t%3", operands);

  p = "p%Z0\t%0";
  strcat (buf, p + !(stack_top_dies || dimode_p));

  output_asm_insn (buf, operands);

  if (round_mode != I387_CW_ANY)
    output_asm_insn ("fldcw\t%2", operands);

  return "";
}

/* Output code for x87 ffreep insn.  The OPNO argument, which may only
   have the values zero or one, indicates the ffreep insn's operand
   from the OPERANDS array.  */

static const char *
output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
{
  if (TARGET_USE_FFREEP)
#ifdef HAVE_AS_IX86_FFREEP
    return opno ? "ffreep\t%y1" : "ffreep\t%y0";
#else
    {
      static char retval[32];
      int regno = REGNO (operands[opno]);

      gcc_assert (STACK_REGNO_P (regno));

      regno -= FIRST_STACK_REG;

      snprintf (retval, sizeof (retval), ASM_SHORT "0xc%ddf", regno);
      return retval;
    }
#endif

  return opno ? "fstp\t%y1" : "fstp\t%y0";
}


/* Output code for INSN to compare OPERANDS.  EFLAGS_P is 1 when fcomi
   should be used.  UNORDERED_P is true when fucom should be used.  */

const char *
output_fp_compare (rtx_insn *insn, rtx *operands,
		   bool eflags_p, bool unordered_p)
{
  rtx *xops = eflags_p ? &operands[0] : &operands[1];
  bool stack_top_dies;

  static char buf[40];
  const char *p;

  gcc_assert (STACK_TOP_P (xops[0]));

  stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG);

  if (eflags_p)
    {
      p = unordered_p ? "fucomi" : "fcomi";
      strcpy (buf, p);

      p = "p\t{%y1, %0|%0, %y1}";
      strcat (buf, p + !stack_top_dies);

      return buf;
    }

  if (STACK_REG_P (xops[1])
      && stack_top_dies
      && find_regno_note (insn, REG_DEAD, FIRST_STACK_REG + 1))
    {
      gcc_assert (REGNO (xops[1]) == FIRST_STACK_REG + 1);

      /* If both the top of the 387 stack die, and the other operand
	 is also a stack register that dies, then this must be a
	 `fcompp' float compare.  */
      p = unordered_p ? "fucompp" : "fcompp";
      strcpy (buf, p);
    }
  else if (const0_operand (xops[1], VOIDmode))
    {
      gcc_assert (!unordered_p);
      strcpy (buf, "ftst");
    }
  else
    {
      if (GET_MODE_CLASS (GET_MODE (xops[1])) == MODE_INT)
	{
	  gcc_assert (!unordered_p);
	  p = "ficom";
	}
      else
	p = unordered_p ? "fucom" : "fcom";

      strcpy (buf, p);

      p = "p%Z2\t%y2";
      strcat (buf, p + !stack_top_dies);
    }

  output_asm_insn (buf, operands);
  return "fnstsw\t%0";
}

void
ix86_output_addr_vec_elt (FILE *file, int value)
{
  const char *directive = ASM_LONG;

#ifdef ASM_QUAD
  if (TARGET_LP64)
    directive = ASM_QUAD;
#else
  gcc_assert (!TARGET_64BIT);
#endif

  fprintf (file, "%s%s%d\n", directive, LPREFIX, value);
}

void
ix86_output_addr_diff_elt (FILE *file, int value, int rel)
{
  const char *directive = ASM_LONG;

#ifdef ASM_QUAD
  if (TARGET_64BIT && CASE_VECTOR_MODE == DImode)
    directive = ASM_QUAD;
#else
  gcc_assert (!TARGET_64BIT);
#endif
  /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand.  */
  if (TARGET_64BIT || TARGET_VXWORKS_RTP)
    fprintf (file, "%s%s%d-%s%d\n",
	     directive, LPREFIX, value, LPREFIX, rel);
#if TARGET_MACHO
  else if (TARGET_MACHO)
    {
      fprintf (file, ASM_LONG "%s%d-", LPREFIX, value);
      machopic_output_function_base_name (file);
      putc ('\n', file);
    }
#endif
  else if (HAVE_AS_GOTOFF_IN_DATA)
    fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value);
  else
    asm_fprintf (file, ASM_LONG "%U%s+[.-%s%d]\n",
		 GOT_SYMBOL_NAME, LPREFIX, value);
}

#define LEA_MAX_STALL (3)
#define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)

/* Increase given DISTANCE in half-cycles according to
   dependencies between PREV and NEXT instructions.
   Add 1 half-cycle if there is no dependency and
   go to next cycle if there is some dependecy.  */

static unsigned int
increase_distance (rtx_insn *prev, rtx_insn *next, unsigned int distance)
{
  df_ref def, use;

  if (!prev || !next)
    return distance + (distance & 1) + 2;

  if (!DF_INSN_USES (next) || !DF_INSN_DEFS (prev))
    return distance + 1;

  FOR_EACH_INSN_USE (use, next)
    FOR_EACH_INSN_DEF (def, prev)
      if (!DF_REF_IS_ARTIFICIAL (def)
	  && DF_REF_REGNO (use) == DF_REF_REGNO (def))
	return distance + (distance & 1) + 2;

  return distance + 1;
}

/* Function checks if instruction INSN defines register number
   REGNO1 or REGNO2.  */

bool
insn_defines_reg (unsigned int regno1, unsigned int regno2,
		  rtx_insn *insn)
{
  df_ref def;

  FOR_EACH_INSN_DEF (def, insn)
    if (DF_REF_REG_DEF_P (def)
	&& !DF_REF_IS_ARTIFICIAL (def)
	&& (regno1 == DF_REF_REGNO (def)
	    || regno2 == DF_REF_REGNO (def)))
      return true;

  return false;
}

/* Function checks if instruction INSN uses register number
   REGNO as a part of address expression.  */

static bool
insn_uses_reg_mem (unsigned int regno, rtx insn)
{
  df_ref use;

  FOR_EACH_INSN_USE (use, insn)
    if (DF_REF_REG_MEM_P (use) && regno == DF_REF_REGNO (use))
      return true;

  return false;
}

/* Search backward for non-agu definition of register number REGNO1
   or register number REGNO2 in basic block starting from instruction
   START up to head of basic block or instruction INSN.

   Function puts true value into *FOUND var if definition was found
   and false otherwise.

   Distance in half-cycles between START and found instruction or head
   of BB is added to DISTANCE and returned.  */

static int
distance_non_agu_define_in_bb (unsigned int regno1, unsigned int regno2,
			       rtx_insn *insn, int distance,
			       rtx_insn *start, bool *found)
{
  basic_block bb = start ? BLOCK_FOR_INSN (start) : NULL;
  rtx_insn *prev = start;
  rtx_insn *next = NULL;

  *found = false;

  while (prev
	 && prev != insn
	 && distance < LEA_SEARCH_THRESHOLD)
    {
      if (NONDEBUG_INSN_P (prev) && NONJUMP_INSN_P (prev))
	{
	  distance = increase_distance (prev, next, distance);
	  if (insn_defines_reg (regno1, regno2, prev))
	    {
	      if (recog_memoized (prev) < 0
		  || get_attr_type (prev) != TYPE_LEA)
		{
		  *found = true;
		  return distance;
		}
	    }

	  next = prev;
	}
      if (prev == BB_HEAD (bb))
	break;

      prev = PREV_INSN (prev);
    }

  return distance;
}

/* Search backward for non-agu definition of register number REGNO1
   or register number REGNO2 in INSN's basic block until
   1. Pass LEA_SEARCH_THRESHOLD instructions, or
   2. Reach neighbor BBs boundary, or
   3. Reach agu definition.
   Returns the distance between the non-agu definition point and INSN.
   If no definition point, returns -1.  */

static int
distance_non_agu_define (unsigned int regno1, unsigned int regno2,
			 rtx_insn *insn)
{
  basic_block bb = BLOCK_FOR_INSN (insn);
  int distance = 0;
  bool found = false;

  if (insn != BB_HEAD (bb))
    distance = distance_non_agu_define_in_bb (regno1, regno2, insn,
					      distance, PREV_INSN (insn),
					      &found);

  if (!found && distance < LEA_SEARCH_THRESHOLD)
    {
      edge e;
      edge_iterator ei;
      bool simple_loop = false;

      FOR_EACH_EDGE (e, ei, bb->preds)
	if (e->src == bb)
	  {
	    simple_loop = true;
	    break;
	  }

      if (simple_loop)
	distance = distance_non_agu_define_in_bb (regno1, regno2,
						  insn, distance,
						  BB_END (bb), &found);
      else
	{
	  int shortest_dist = -1;
	  bool found_in_bb = false;

	  FOR_EACH_EDGE (e, ei, bb->preds)
	    {
	      int bb_dist
		= distance_non_agu_define_in_bb (regno1, regno2,
						 insn, distance,
						 BB_END (e->src),
						 &found_in_bb);
	      if (found_in_bb)
		{
		  if (shortest_dist < 0)
		    shortest_dist = bb_dist;
		  else if (bb_dist > 0)
		    shortest_dist = MIN (bb_dist, shortest_dist);

		  found = true;
		}
	    }

	  distance = shortest_dist;
	}
    }

  if (!found)
    return -1;

  return distance >> 1;
}

/* Return the distance in half-cycles between INSN and the next
   insn that uses register number REGNO in memory address added
   to DISTANCE.  Return -1 if REGNO0 is set.

   Put true value into *FOUND if register usage was found and
   false otherwise.
   Put true value into *REDEFINED if register redefinition was
   found and false otherwise.  */

static int
distance_agu_use_in_bb (unsigned int regno,
			rtx_insn *insn, int distance, rtx_insn *start,
			bool *found, bool *redefined)
{
  basic_block bb = NULL;
  rtx_insn *next = start;
  rtx_insn *prev = NULL;

  *found = false;
  *redefined = false;

  if (start != NULL_RTX)
    {
      bb = BLOCK_FOR_INSN (start);
      if (start != BB_HEAD (bb))
	/* If insn and start belong to the same bb, set prev to insn,
	   so the call to increase_distance will increase the distance
	   between insns by 1.  */
	prev = insn;
    }

  while (next
	 && next != insn
	 && distance < LEA_SEARCH_THRESHOLD)
    {
      if (NONDEBUG_INSN_P (next) && NONJUMP_INSN_P (next))
	{
	  distance = increase_distance(prev, next, distance);
	  if (insn_uses_reg_mem (regno, next))
	    {
	      /* Return DISTANCE if OP0 is used in memory
		 address in NEXT.  */
	      *found = true;
	      return distance;
	    }

	  if (insn_defines_reg (regno, INVALID_REGNUM, next))
	    {
	      /* Return -1 if OP0 is set in NEXT.  */
	      *redefined = true;
	      return -1;
	    }

	  prev = next;
	}

      if (next == BB_END (bb))
	break;

      next = NEXT_INSN (next);
    }

  return distance;
}

/* Return the distance between INSN and the next insn that uses
   register number REGNO0 in memory address.  Return -1 if no such
   a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set.  */

static int
distance_agu_use (unsigned int regno0, rtx_insn *insn)
{
  basic_block bb = BLOCK_FOR_INSN (insn);
  int distance = 0;
  bool found = false;
  bool redefined = false;

  if (insn != BB_END (bb))
    distance = distance_agu_use_in_bb (regno0, insn, distance,
				       NEXT_INSN (insn),
				       &found, &redefined);

  if (!found && !redefined && distance < LEA_SEARCH_THRESHOLD)
    {
      edge e;
      edge_iterator ei;
      bool simple_loop = false;

      FOR_EACH_EDGE (e, ei, bb->succs)
        if (e->dest == bb)
	  {
	    simple_loop = true;
	    break;
	  }

      if (simple_loop)
	distance = distance_agu_use_in_bb (regno0, insn,
					   distance, BB_HEAD (bb),
					   &found, &redefined);
      else
	{
	  int shortest_dist = -1;
	  bool found_in_bb = false;
	  bool redefined_in_bb = false;

	  FOR_EACH_EDGE (e, ei, bb->succs)
	    {
	      int bb_dist
		= distance_agu_use_in_bb (regno0, insn,
					  distance, BB_HEAD (e->dest),
					  &found_in_bb, &redefined_in_bb);
	      if (found_in_bb)
		{
		  if (shortest_dist < 0)
		    shortest_dist = bb_dist;
		  else if (bb_dist > 0)
		    shortest_dist = MIN (bb_dist, shortest_dist);

		  found = true;
		}
	    }

	  distance = shortest_dist;
	}
    }

  if (!found || redefined)
    return -1;

  return distance >> 1;
}

/* Define this macro to tune LEA priority vs ADD, it take effect when
   there is a dilemma of choosing LEA or ADD
   Negative value: ADD is more preferred than LEA
   Zero: Neutral
   Positive value: LEA is more preferred than ADD.  */
#define IX86_LEA_PRIORITY 0

/* Return true if usage of lea INSN has performance advantage
   over a sequence of instructions.  Instructions sequence has
   SPLIT_COST cycles higher latency than lea latency.  */

static bool
ix86_lea_outperforms (rtx_insn *insn, unsigned int regno0, unsigned int regno1,
		      unsigned int regno2, int split_cost, bool has_scale)
{
  int dist_define, dist_use;

  /* For Atom processors newer than Bonnell, if using a 2-source or
     3-source LEA for non-destructive destination purposes, or due to
     wanting ability to use SCALE, the use of LEA is justified.  */
  if (!TARGET_CPU_P (BONNELL))
    {
      if (has_scale)
	return true;
      if (split_cost < 1)
	return false;
      if (regno0 == regno1 || regno0 == regno2)
	return false;
      return true;
    }

  /* Remember recog_data content.  */
  struct recog_data_d recog_data_save = recog_data;

  dist_define = distance_non_agu_define (regno1, regno2, insn);
  dist_use = distance_agu_use (regno0, insn);

  /* distance_non_agu_define can call get_attr_type which can call
     recog_memoized, restore recog_data back to previous content.  */
  recog_data = recog_data_save;

  if (dist_define < 0 || dist_define >= LEA_MAX_STALL)
    {
      /* If there is no non AGU operand definition, no AGU
	 operand usage and split cost is 0 then both lea
	 and non lea variants have same priority.  Currently
	 we prefer lea for 64 bit code and non lea on 32 bit
	 code.  */
      if (dist_use < 0 && split_cost == 0)
	return TARGET_64BIT || IX86_LEA_PRIORITY;
      else
	return true;
    }

  /* With longer definitions distance lea is more preferable.
     Here we change it to take into account splitting cost and
     lea priority.  */
  dist_define += split_cost + IX86_LEA_PRIORITY;

  /* If there is no use in memory addess then we just check
     that split cost exceeds AGU stall.  */
  if (dist_use < 0)
    return dist_define > LEA_MAX_STALL;

  /* If this insn has both backward non-agu dependence and forward
     agu dependence, the one with short distance takes effect.  */
  return dist_define >= dist_use;
}

/* Return true if we need to split op0 = op1 + op2 into a sequence of
   move and add to avoid AGU stalls.  */

bool
ix86_avoid_lea_for_add (rtx_insn *insn, rtx operands[])
{
  unsigned int regno0, regno1, regno2;

  /* Check if we need to optimize.  */
  if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
    return false;

  regno0 = true_regnum (operands[0]);
  regno1 = true_regnum (operands[1]);
  regno2 = true_regnum (operands[2]);

  /* We need to split only adds with non destructive
     destination operand.  */
  if (regno0 == regno1 || regno0 == regno2)
    return false;
  else
    return !ix86_lea_outperforms (insn, regno0, regno1, regno2, 1, false);
}

/* Return true if we should emit lea instruction instead of mov
   instruction.  */

bool
ix86_use_lea_for_mov (rtx_insn *insn, rtx operands[])
{
  unsigned int regno0, regno1;

  /* Check if we need to optimize.  */
  if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
    return false;

  /* Use lea for reg to reg moves only.  */
  if (!REG_P (operands[0]) || !REG_P (operands[1]))
    return false;

  regno0 = true_regnum (operands[0]);
  regno1 = true_regnum (operands[1]);

  return ix86_lea_outperforms (insn, regno0, regno1, INVALID_REGNUM, 0, false);
}

/* Return true if we need to split lea into a sequence of
   instructions to avoid AGU stalls during peephole2. */

bool
ix86_avoid_lea_for_addr (rtx_insn *insn, rtx operands[])
{
  unsigned int regno0, regno1, regno2;
  int split_cost;
  struct ix86_address parts;
  int ok;

  /* The "at least two components" test below might not catch simple
     move or zero extension insns if parts.base is non-NULL and parts.disp
     is const0_rtx as the only components in the address, e.g. if the
     register is %rbp or %r13.  As this test is much cheaper and moves or
     zero extensions are the common case, do this check first.  */
  if (REG_P (operands[1])
      || (SImode_address_operand (operands[1], VOIDmode)
	  && REG_P (XEXP (operands[1], 0))))
    return false;

  ok = ix86_decompose_address (operands[1], &parts);
  gcc_assert (ok);

  /* There should be at least two components in the address.  */
  if ((parts.base != NULL_RTX) + (parts.index != NULL_RTX)
      + (parts.disp != NULL_RTX) + (parts.scale > 1) < 2)
    return false;

  /* We should not split into add if non legitimate pic
     operand is used as displacement. */
  if (parts.disp && flag_pic && !LEGITIMATE_PIC_OPERAND_P (parts.disp))
    return false;

  regno0 = true_regnum (operands[0]) ;
  regno1 = INVALID_REGNUM;
  regno2 = INVALID_REGNUM;

  if (parts.base)
    regno1 = true_regnum (parts.base);
  if (parts.index)
    regno2 = true_regnum (parts.index);

  /* Use add for a = a + b and a = b + a since it is faster and shorter
     than lea for most processors.  For the processors like BONNELL, if
     the destination register of LEA holds an actual address which will
     be used soon, LEA is better and otherwise ADD is better.  */
  if (!TARGET_CPU_P (BONNELL)
      && parts.scale == 1
      && (!parts.disp || parts.disp == const0_rtx)
      && (regno0 == regno1 || regno0 == regno2))
    return true;

  /* Split with -Oz if the encoding requires fewer bytes.  */
  if (optimize_size > 1
      && parts.scale > 1
      && !parts.base
      && (!parts.disp || parts.disp == const0_rtx)) 
    return true;

  /* Check we need to optimize.  */
  if (!TARGET_AVOID_LEA_FOR_ADDR || optimize_function_for_size_p (cfun))
    return false;

  split_cost = 0;

  /* Compute how many cycles we will add to execution time
     if split lea into a sequence of instructions.  */
  if (parts.base || parts.index)
    {
      /* Have to use mov instruction if non desctructive
	 destination form is used.  */
      if (regno1 != regno0 && regno2 != regno0)
	split_cost += 1;

      /* Have to add index to base if both exist.  */
      if (parts.base && parts.index)
	split_cost += 1;

      /* Have to use shift and adds if scale is 2 or greater.  */
      if (parts.scale > 1)
	{
	  if (regno0 != regno1)
	    split_cost += 1;
	  else if (regno2 == regno0)
	    split_cost += 4;
	  else
	    split_cost += parts.scale;
	}

      /* Have to use add instruction with immediate if
	 disp is non zero.  */
      if (parts.disp && parts.disp != const0_rtx)
	split_cost += 1;

      /* Subtract the price of lea.  */
      split_cost -= 1;
    }

  return !ix86_lea_outperforms (insn, regno0, regno1, regno2, split_cost,
				parts.scale > 1);
}

/* Return true if it is ok to optimize an ADD operation to LEA
   operation to avoid flag register consumation.  For most processors,
   ADD is faster than LEA.  For the processors like BONNELL, if the
   destination register of LEA holds an actual address which will be
   used soon, LEA is better and otherwise ADD is better.  */

bool
ix86_lea_for_add_ok (rtx_insn *insn, rtx operands[])
{
  unsigned int regno0 = true_regnum (operands[0]);
  unsigned int regno1 = true_regnum (operands[1]);
  unsigned int regno2 = true_regnum (operands[2]);

  /* If a = b + c, (a!=b && a!=c), must use lea form. */
  if (regno0 != regno1 && regno0 != regno2)
    return true;

  if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
    return false;

  return ix86_lea_outperforms (insn, regno0, regno1, regno2, 0, false);
}

/* Return true if destination reg of SET_BODY is shift count of
   USE_BODY.  */

static bool
ix86_dep_by_shift_count_body (const_rtx set_body, const_rtx use_body)
{
  rtx set_dest;
  rtx shift_rtx;
  int i;

  /* Retrieve destination of SET_BODY.  */
  switch (GET_CODE (set_body))
    {
    case SET:
      set_dest = SET_DEST (set_body);
      if (!set_dest || !REG_P (set_dest))
	return false;
      break;
    case PARALLEL:
      for (i = XVECLEN (set_body, 0) - 1; i >= 0; i--)
	if (ix86_dep_by_shift_count_body (XVECEXP (set_body, 0, i),
					  use_body))
	  return true;
      /* FALLTHROUGH */
    default:
      return false;
    }

  /* Retrieve shift count of USE_BODY.  */
  switch (GET_CODE (use_body))
    {
    case SET:
      shift_rtx = XEXP (use_body, 1);
      break;
    case PARALLEL:
      for (i = XVECLEN (use_body, 0) - 1; i >= 0; i--)
	if (ix86_dep_by_shift_count_body (set_body,
					  XVECEXP (use_body, 0, i)))
	  return true;
      /* FALLTHROUGH */
    default:
      return false;
    }

  if (shift_rtx
      && (GET_CODE (shift_rtx) == ASHIFT
	  || GET_CODE (shift_rtx) == LSHIFTRT
	  || GET_CODE (shift_rtx) == ASHIFTRT
	  || GET_CODE (shift_rtx) == ROTATE
	  || GET_CODE (shift_rtx) == ROTATERT))
    {
      rtx shift_count = XEXP (shift_rtx, 1);

      /* Return true if shift count is dest of SET_BODY.  */
      if (REG_P (shift_count))
	{
	  /* Add check since it can be invoked before register
	     allocation in pre-reload schedule.  */
	  if (reload_completed
	      && true_regnum (set_dest) == true_regnum (shift_count))
	    return true;
	  else if (REGNO(set_dest) == REGNO(shift_count))
	    return true;
	}
    }

  return false;
}

/* Return true if destination reg of SET_INSN is shift count of
   USE_INSN.  */

bool
ix86_dep_by_shift_count (const_rtx set_insn, const_rtx use_insn)
{
  return ix86_dep_by_shift_count_body (PATTERN (set_insn),
				       PATTERN (use_insn));
}

/* Return TRUE or FALSE depending on whether the unary operator meets the
   appropriate constraints.  */

bool
ix86_unary_operator_ok (enum rtx_code,
			machine_mode,
			rtx operands[2])
{
  /* If one of operands is memory, source and destination must match.  */
  if ((MEM_P (operands[0])
       || MEM_P (operands[1]))
      && ! rtx_equal_p (operands[0], operands[1]))
    return false;
  return true;
}

/* Return TRUE if the operands to a vec_interleave_{high,low}v2df
   are ok, keeping in mind the possible movddup alternative.  */

bool
ix86_vec_interleave_v2df_operator_ok (rtx operands[3], bool high)
{
  if (MEM_P (operands[0]))
    return rtx_equal_p (operands[0], operands[1 + high]);
  if (MEM_P (operands[1]) && MEM_P (operands[2]))
    return false;
  return true;
}

/* A subroutine of ix86_build_signbit_mask.  If VECT is true,
   then replicate the value for all elements of the vector
   register.  */

rtx
ix86_build_const_vector (machine_mode mode, bool vect, rtx value)
{
  int i, n_elt;
  rtvec v;
  machine_mode scalar_mode;

  switch (mode)
    {
    case E_V64QImode:
    case E_V32QImode:
    case E_V16QImode:
    case E_V32HImode:
    case E_V16HImode:
    case E_V8HImode:
    case E_V16SImode:
    case E_V8SImode:
    case E_V4SImode:
    case E_V2SImode:
    case E_V8DImode:
    case E_V4DImode:
    case E_V2DImode:
      gcc_assert (vect);
      /* FALLTHRU */
    case E_V8HFmode:
    case E_V16HFmode:
    case E_V32HFmode:
    case E_V16SFmode:
    case E_V8SFmode:
    case E_V4SFmode:
    case E_V2SFmode:
    case E_V8DFmode:
    case E_V4DFmode:
    case E_V2DFmode:
      n_elt = GET_MODE_NUNITS (mode);
      v = rtvec_alloc (n_elt);
      scalar_mode = GET_MODE_INNER (mode);

      RTVEC_ELT (v, 0) = value;

      for (i = 1; i < n_elt; ++i)
	RTVEC_ELT (v, i) = vect ? value : CONST0_RTX (scalar_mode);

      return gen_rtx_CONST_VECTOR (mode, v);

    default:
      gcc_unreachable ();
    }
}

/* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
   and ix86_expand_int_vcond.  Create a mask for the sign bit in MODE
   for an SSE register.  If VECT is true, then replicate the mask for
   all elements of the vector register.  If INVERT is true, then create
   a mask excluding the sign bit.  */

rtx
ix86_build_signbit_mask (machine_mode mode, bool vect, bool invert)
{
  machine_mode vec_mode, imode;
  wide_int w;
  rtx mask, v;

  switch (mode)
    {
    case E_V8HFmode:
    case E_V16HFmode:
    case E_V32HFmode:
      vec_mode = mode;
      imode = HImode;
      break;

    case E_V16SImode:
    case E_V16SFmode:
    case E_V8SImode:
    case E_V4SImode:
    case E_V8SFmode:
    case E_V4SFmode:
    case E_V2SFmode:
    case E_V2SImode:
      vec_mode = mode;
      imode = SImode;
      break;

    case E_V8DImode:
    case E_V4DImode:
    case E_V2DImode:
    case E_V8DFmode:
    case E_V4DFmode:
    case E_V2DFmode:
      vec_mode = mode;
      imode = DImode;
      break;

    case E_TImode:
    case E_TFmode:
      vec_mode = VOIDmode;
      imode = TImode;
      break;

    default:
      gcc_unreachable ();
    }

  machine_mode inner_mode = GET_MODE_INNER (mode);
  w = wi::set_bit_in_zero (GET_MODE_BITSIZE (inner_mode) - 1,
			   GET_MODE_BITSIZE (inner_mode));
  if (invert)
    w = wi::bit_not (w);

  /* Force this value into the low part of a fp vector constant.  */
  mask = immed_wide_int_const (w, imode);
  mask = gen_lowpart (inner_mode, mask);

  if (vec_mode == VOIDmode)
    return force_reg (inner_mode, mask);

  v = ix86_build_const_vector (vec_mode, vect, mask);
  return force_reg (vec_mode, v);
}

/* Return HOST_WIDE_INT for const vector OP in MODE.  */

HOST_WIDE_INT
ix86_convert_const_vector_to_integer (rtx op, machine_mode mode)
{
  if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
    gcc_unreachable ();

  int nunits = GET_MODE_NUNITS (mode);
  wide_int val = wi::zero (GET_MODE_BITSIZE (mode));
  machine_mode innermode = GET_MODE_INNER (mode);
  unsigned int innermode_bits = GET_MODE_BITSIZE (innermode);

  switch (mode)
    {
    case E_V2QImode:
    case E_V4QImode:
    case E_V2HImode:
    case E_V8QImode:
    case E_V4HImode:
    case E_V2SImode:
      for (int i = 0; i < nunits; ++i)
	{
	  int v = INTVAL (XVECEXP (op, 0, i));
	  wide_int wv = wi::shwi (v, innermode_bits);
	  val = wi::insert (val, wv, innermode_bits * i, innermode_bits);
	}
      break;
    case E_V2HFmode:
    case E_V2BFmode:
    case E_V4HFmode:
    case E_V4BFmode:
    case E_V2SFmode:
      for (int i = 0; i < nunits; ++i)
	{
	  rtx x = XVECEXP (op, 0, i);
	  int v = real_to_target (NULL, CONST_DOUBLE_REAL_VALUE (x),
				  REAL_MODE_FORMAT (innermode));
	  wide_int wv = wi::shwi (v, innermode_bits);
	  val = wi::insert (val, wv, innermode_bits * i, innermode_bits);
	}
      break;
    default:
      gcc_unreachable ();
    }

  return val.to_shwi ();
}

/* Return TRUE or FALSE depending on whether the first SET in INSN
   has source and destination with matching CC modes, and that the
   CC mode is at least as constrained as REQ_MODE.  */

bool
ix86_match_ccmode (rtx insn, machine_mode req_mode)
{
  rtx set;
  machine_mode set_mode;

  set = PATTERN (insn);
  if (GET_CODE (set) == PARALLEL)
    set = XVECEXP (set, 0, 0);
  gcc_assert (GET_CODE (set) == SET);
  gcc_assert (GET_CODE (SET_SRC (set)) == COMPARE);

  set_mode = GET_MODE (SET_DEST (set));
  switch (set_mode)
    {
    case E_CCNOmode:
      if (req_mode != CCNOmode
	  && (req_mode != CCmode
	      || XEXP (SET_SRC (set), 1) != const0_rtx))
	return false;
      break;
    case E_CCmode:
      if (req_mode == CCGCmode)
	return false;
      /* FALLTHRU */
    case E_CCGCmode:
      if (req_mode == CCGOCmode || req_mode == CCNOmode)
	return false;
      /* FALLTHRU */
    case E_CCGOCmode:
      if (req_mode == CCZmode)
	return false;
      /* FALLTHRU */
    case E_CCZmode:
      break;

    case E_CCGZmode:

    case E_CCAmode:
    case E_CCCmode:
    case E_CCOmode:
    case E_CCPmode:
    case E_CCSmode:
      if (set_mode != req_mode)
	return false;
      break;

    default:
      gcc_unreachable ();
    }

  return GET_MODE (SET_SRC (set)) == set_mode;
}

machine_mode
ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
{
  machine_mode mode = GET_MODE (op0);

  if (SCALAR_FLOAT_MODE_P (mode))
    {
      gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
      return CCFPmode;
    }

  switch (code)
    {
      /* Only zero flag is needed.  */
    case EQ:			/* ZF=0 */
    case NE:			/* ZF!=0 */
      return CCZmode;
      /* Codes needing carry flag.  */
    case GEU:			/* CF=0 */
    case LTU:			/* CF=1 */
      rtx geu;
      /* Detect overflow checks.  They need just the carry flag.  */
      if (GET_CODE (op0) == PLUS
	  && (rtx_equal_p (op1, XEXP (op0, 0))
	      || rtx_equal_p (op1, XEXP (op0, 1))))
	return CCCmode;
      /* Similarly for *setcc_qi_addqi3_cconly_overflow_1_* patterns.
	 Match LTU of op0
	 (neg:QI (geu:QI (reg:CC_CCC FLAGS_REG) (const_int 0)))
	 and op1
	 (ltu:QI (reg:CC_CCC FLAGS_REG) (const_int 0))
	 where CC_CCC is either CC or CCC.  */
      else if (code == LTU
	       && GET_CODE (op0) == NEG
	       && GET_CODE (geu = XEXP (op0, 0)) == GEU
	       && REG_P (XEXP (geu, 0))
	       && (GET_MODE (XEXP (geu, 0)) == CCCmode
		   || GET_MODE (XEXP (geu, 0)) == CCmode)
	       && REGNO (XEXP (geu, 0)) == FLAGS_REG
	       && XEXP (geu, 1) == const0_rtx
	       && GET_CODE (op1) == LTU
	       && REG_P (XEXP (op1, 0))
	       && GET_MODE (XEXP (op1, 0)) == GET_MODE (XEXP (geu, 0))
	       && REGNO (XEXP (op1, 0)) == FLAGS_REG
	       && XEXP (op1, 1) == const0_rtx)
	return CCCmode;
      /* Similarly for *x86_cmc pattern.
	 Match LTU of op0 (neg:QI (ltu:QI (reg:CCC FLAGS_REG) (const_int 0)))
	 and op1 (geu:QI (reg:CCC FLAGS_REG) (const_int 0)).
	 It is sufficient to test that the operand modes are CCCmode.  */
      else if (code == LTU
	       && GET_CODE (op0) == NEG
	       && GET_CODE (XEXP (op0, 0)) == LTU
	       && GET_MODE (XEXP (XEXP (op0, 0), 0)) == CCCmode
	       && GET_CODE (op1) == GEU
	       && GET_MODE (XEXP (op1, 0)) == CCCmode)
	return CCCmode;
      else
	return CCmode;
    case GTU:			/* CF=0 & ZF=0 */
    case LEU:			/* CF=1 | ZF=1 */
      return CCmode;
      /* Codes possibly doable only with sign flag when
         comparing against zero.  */
    case GE:			/* SF=OF   or   SF=0 */
    case LT:			/* SF<>OF  or   SF=1 */
      if (op1 == const0_rtx)
	return CCGOCmode;
      else
	/* For other cases Carry flag is not required.  */
	return CCGCmode;
      /* Codes doable only with sign flag when comparing
         against zero, but we miss jump instruction for it
         so we need to use relational tests against overflow
         that thus needs to be zero.  */
    case GT:			/* ZF=0 & SF=OF */
    case LE:			/* ZF=1 | SF<>OF */
      if (op1 == const0_rtx)
	return CCNOmode;
      else
	return CCGCmode;
      /* strcmp pattern do (use flags) and combine may ask us for proper
	 mode.  */
    case USE:
      return CCmode;
    default:
      gcc_unreachable ();
    }
}

/* Return TRUE or FALSE depending on whether the ptest instruction
   INSN has source and destination with suitable matching CC modes.  */

bool
ix86_match_ptest_ccmode (rtx insn)
{
  rtx set, src;
  machine_mode set_mode;

  set = PATTERN (insn);
  gcc_assert (GET_CODE (set) == SET);
  src = SET_SRC (set);
  gcc_assert (GET_CODE (src) == UNSPEC
	      && XINT (src, 1) == UNSPEC_PTEST);

  set_mode = GET_MODE (src);
  if (set_mode != CCZmode
      && set_mode != CCCmode
      && set_mode != CCmode)
    return false;
  return GET_MODE (SET_DEST (set)) == set_mode;
}

/* Return the fixed registers used for condition codes.  */

static bool
ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2)
{
  *p1 = FLAGS_REG;
  *p2 = INVALID_REGNUM;
  return true;
}

/* If two condition code modes are compatible, return a condition code
   mode which is compatible with both.  Otherwise, return
   VOIDmode.  */

static machine_mode
ix86_cc_modes_compatible (machine_mode m1, machine_mode m2)
{
  if (m1 == m2)
    return m1;

  if (GET_MODE_CLASS (m1) != MODE_CC || GET_MODE_CLASS (m2) != MODE_CC)
    return VOIDmode;

  if ((m1 == CCGCmode && m2 == CCGOCmode)
      || (m1 == CCGOCmode && m2 == CCGCmode))
    return CCGCmode;

  if ((m1 == CCNOmode && m2 == CCGOCmode)
      || (m1 == CCGOCmode && m2 == CCNOmode))
    return CCNOmode;

  if (m1 == CCZmode
      && (m2 == CCGCmode || m2 == CCGOCmode || m2 == CCNOmode))
    return m2;
  else if (m2 == CCZmode
	   && (m1 == CCGCmode || m1 == CCGOCmode || m1 == CCNOmode))
    return m1;

  switch (m1)
    {
    default:
      gcc_unreachable ();

    case E_CCmode:
    case E_CCGCmode:
    case E_CCGOCmode:
    case E_CCNOmode:
    case E_CCAmode:
    case E_CCCmode:
    case E_CCOmode:
    case E_CCPmode:
    case E_CCSmode:
    case E_CCZmode:
      switch (m2)
	{
	default:
	  return VOIDmode;

	case E_CCmode:
	case E_CCGCmode:
	case E_CCGOCmode:
	case E_CCNOmode:
	case E_CCAmode:
	case E_CCCmode:
	case E_CCOmode:
	case E_CCPmode:
	case E_CCSmode:
	case E_CCZmode:
	  return CCmode;
	}

    case E_CCFPmode:
      /* These are only compatible with themselves, which we already
	 checked above.  */
      return VOIDmode;
    }
}

/* Return strategy to use for floating-point.  We assume that fcomi is always
   preferrable where available, since that is also true when looking at size
   (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test).  */

enum ix86_fpcmp_strategy
ix86_fp_comparison_strategy (enum rtx_code)
{
  /* Do fcomi/sahf based test when profitable.  */

  if (TARGET_CMOVE)
    return IX86_FPCMP_COMI;

  if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
    return IX86_FPCMP_SAHF;

  return IX86_FPCMP_ARITH;
}

/* Convert comparison codes we use to represent FP comparison to integer
   code that will result in proper branch.  Return UNKNOWN if no such code
   is available.  */

enum rtx_code
ix86_fp_compare_code_to_integer (enum rtx_code code)
{
  switch (code)
    {
    case GT:
      return GTU;
    case GE:
      return GEU;
    case ORDERED:
    case UNORDERED:
      return code;
    case UNEQ:
      return EQ;
    case UNLT:
      return LTU;
    case UNLE:
      return LEU;
    case LTGT:
      return NE;
    default:
      return UNKNOWN;
    }
}

/* Zero extend possibly SImode EXP to Pmode register.  */
rtx
ix86_zero_extend_to_Pmode (rtx exp)
{
  return force_reg (Pmode, convert_to_mode (Pmode, exp, 1));
}

/* Return true if the function is called via PLT.   */

bool
ix86_call_use_plt_p (rtx call_op)
{
  if (SYMBOL_REF_LOCAL_P (call_op))
    {
      if (SYMBOL_REF_DECL (call_op)
	  && TREE_CODE (SYMBOL_REF_DECL (call_op)) == FUNCTION_DECL)
	{
	  /* NB: All ifunc functions must be called via PLT.  */
	  cgraph_node *node
	    = cgraph_node::get (SYMBOL_REF_DECL (call_op));
	  if (node && node->ifunc_resolver)
	    return true;
	}
      return false;
    }
  return true;
}

/* Implement TARGET_IFUNC_REF_LOCAL_OK.  If this hook returns true,
   the PLT entry will be used as the function address for local IFUNC
   functions.  When the PIC register is needed for PLT call, indirect
   call via the PLT entry will fail since the PIC register may not be
   set up properly for indirect call.  In this case, we should return
   false.  */

static bool
ix86_ifunc_ref_local_ok (void)
{
  return !flag_pic || (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC);
}

/* Return true if the function being called was marked with attribute
   "noplt" or using -fno-plt and we are compiling for non-PIC.  We need
   to handle the non-PIC case in the backend because there is no easy
   interface for the front-end to force non-PLT calls to use the GOT.
   This is currently used only with 64-bit or 32-bit GOT32X ELF targets
   to call the function marked "noplt" indirectly.  */

static bool
ix86_nopic_noplt_attribute_p (rtx call_op)
{
  if (flag_pic || ix86_cmodel == CM_LARGE
      || !(TARGET_64BIT || HAVE_AS_IX86_GOT32X)
      || TARGET_MACHO || TARGET_SEH || TARGET_PECOFF
      || SYMBOL_REF_LOCAL_P (call_op))
    return false;

  tree symbol_decl = SYMBOL_REF_DECL (call_op);

  if (!flag_plt
      || (symbol_decl != NULL_TREE
          && lookup_attribute ("noplt", DECL_ATTRIBUTES (symbol_decl))))
    return true;

  return false;
}

/* Helper to output the jmp/call.  */
static void
ix86_output_jmp_thunk_or_indirect (const char *thunk_name, const int regno)
{
  if (thunk_name != NULL)
    {
      if ((REX_INT_REGNO_P (regno) || REX2_INT_REGNO_P (regno))
	  && ix86_indirect_branch_cs_prefix)
	fprintf (asm_out_file, "\tcs\n");
      fprintf (asm_out_file, "\tjmp\t");
      assemble_name (asm_out_file, thunk_name);
      putc ('\n', asm_out_file);
      if ((ix86_harden_sls & harden_sls_indirect_jmp))
	fputs ("\tint3\n", asm_out_file);
    }
  else
    output_indirect_thunk (regno);
}

/* Output indirect branch via a call and return thunk.  CALL_OP is a
   register which contains the branch target.  XASM is the assembly
   template for CALL_OP.  Branch is a tail call if SIBCALL_P is true.
   A normal call is converted to:

	call __x86_indirect_thunk_reg

   and a tail call is converted to:

	jmp __x86_indirect_thunk_reg
 */

static void
ix86_output_indirect_branch_via_reg (rtx call_op, bool sibcall_p)
{
  char thunk_name_buf[32];
  char *thunk_name;
  enum indirect_thunk_prefix need_prefix
    = indirect_thunk_need_prefix (current_output_insn);
  int regno = REGNO (call_op);

  if (cfun->machine->indirect_branch_type
      != indirect_branch_thunk_inline)
    {
      if (cfun->machine->indirect_branch_type == indirect_branch_thunk)
	SET_HARD_REG_BIT (indirect_thunks_used, regno);

      indirect_thunk_name (thunk_name_buf, regno, need_prefix, false);
      thunk_name = thunk_name_buf;
    }
  else
    thunk_name = NULL;

  if (sibcall_p)
     ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
  else
    {
      if (thunk_name != NULL)
	{
	  if ((REX_INT_REGNO_P (regno) || REX_INT_REGNO_P (regno))
	      && ix86_indirect_branch_cs_prefix)
	    fprintf (asm_out_file, "\tcs\n");
	  fprintf (asm_out_file, "\tcall\t");
	  assemble_name (asm_out_file, thunk_name);
	  putc ('\n', asm_out_file);
	  return;
	}

      char indirectlabel1[32];
      char indirectlabel2[32];

      ASM_GENERATE_INTERNAL_LABEL (indirectlabel1,
				   INDIRECT_LABEL,
				   indirectlabelno++);
      ASM_GENERATE_INTERNAL_LABEL (indirectlabel2,
				   INDIRECT_LABEL,
				   indirectlabelno++);

      /* Jump.  */
      fputs ("\tjmp\t", asm_out_file);
      assemble_name_raw (asm_out_file, indirectlabel2);
      fputc ('\n', asm_out_file);

      ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);

     ix86_output_jmp_thunk_or_indirect (thunk_name, regno);

      ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);

      /* Call.  */
      fputs ("\tcall\t", asm_out_file);
      assemble_name_raw (asm_out_file, indirectlabel1);
      fputc ('\n', asm_out_file);
    }
}

/* Output indirect branch via a call and return thunk.  CALL_OP is
   the branch target.  XASM is the assembly template for CALL_OP.
   Branch is a tail call if SIBCALL_P is true.  A normal call is
   converted to:

	jmp L2
   L1:
	push CALL_OP
	jmp __x86_indirect_thunk
   L2:
	call L1

   and a tail call is converted to:

	push CALL_OP
	jmp __x86_indirect_thunk
 */

static void
ix86_output_indirect_branch_via_push (rtx call_op, const char *xasm,
				      bool sibcall_p)
{
  char thunk_name_buf[32];
  char *thunk_name;
  char push_buf[64];
  enum indirect_thunk_prefix need_prefix
    = indirect_thunk_need_prefix (current_output_insn);
  int regno = -1;

  if (cfun->machine->indirect_branch_type
      != indirect_branch_thunk_inline)
    {
      if (cfun->machine->indirect_branch_type == indirect_branch_thunk)
	indirect_thunk_needed = true;
      indirect_thunk_name (thunk_name_buf, regno, need_prefix, false);
      thunk_name = thunk_name_buf;
    }
  else
    thunk_name = NULL;

  snprintf (push_buf, sizeof (push_buf), "push{%c}\t%s",
	    TARGET_64BIT ? 'q' : 'l', xasm);

  if (sibcall_p)
    {
      output_asm_insn (push_buf, &call_op);
      ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
    }
  else
    {
      char indirectlabel1[32];
      char indirectlabel2[32];

      ASM_GENERATE_INTERNAL_LABEL (indirectlabel1,
				   INDIRECT_LABEL,
				   indirectlabelno++);
      ASM_GENERATE_INTERNAL_LABEL (indirectlabel2,
				   INDIRECT_LABEL,
				   indirectlabelno++);

      /* Jump.  */
      fputs ("\tjmp\t", asm_out_file);
      assemble_name_raw (asm_out_file, indirectlabel2);
      fputc ('\n', asm_out_file);

      ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);

      /* An external function may be called via GOT, instead of PLT.  */
      if (MEM_P (call_op))
	{
	  struct ix86_address parts;
	  rtx addr = XEXP (call_op, 0);
	  if (ix86_decompose_address (addr, &parts)
	      && parts.base == stack_pointer_rtx)
	    {
	      /* Since call will adjust stack by -UNITS_PER_WORD,
		 we must convert "disp(stack, index, scale)" to
		 "disp+UNITS_PER_WORD(stack, index, scale)".  */
	      if (parts.index)
		{
		  addr = gen_rtx_MULT (Pmode, parts.index,
				       GEN_INT (parts.scale));
		  addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx,
				       addr);
		}
	      else
		addr = stack_pointer_rtx;

	      rtx disp;
	      if (parts.disp != NULL_RTX)
		disp = plus_constant (Pmode, parts.disp,
				      UNITS_PER_WORD);
	      else
		disp = GEN_INT (UNITS_PER_WORD);

	      addr = gen_rtx_PLUS (Pmode, addr, disp);
	      call_op = gen_rtx_MEM (GET_MODE (call_op), addr);
	    }
	}

      output_asm_insn (push_buf, &call_op);

      ix86_output_jmp_thunk_or_indirect (thunk_name, regno);

      ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);

      /* Call.  */
      fputs ("\tcall\t", asm_out_file);
      assemble_name_raw (asm_out_file, indirectlabel1);
      fputc ('\n', asm_out_file);
    }
}

/* Output indirect branch via a call and return thunk.  CALL_OP is
   the branch target.  XASM is the assembly template for CALL_OP.
   Branch is a tail call if SIBCALL_P is true.   */

static void
ix86_output_indirect_branch (rtx call_op, const char *xasm,
			     bool sibcall_p)
{
  if (REG_P (call_op))
    ix86_output_indirect_branch_via_reg (call_op, sibcall_p);
  else
    ix86_output_indirect_branch_via_push (call_op, xasm, sibcall_p);
}

/* Output indirect jump.  CALL_OP is the jump target.  */

const char *
ix86_output_indirect_jmp (rtx call_op)
{
  if (cfun->machine->indirect_branch_type != indirect_branch_keep)
    {
      /* We can't have red-zone since "call" in the indirect thunk
         pushes the return address onto stack, destroying red-zone.  */
      if (ix86_red_zone_used)
	gcc_unreachable ();

      ix86_output_indirect_branch (call_op, "%0", true);
    }
  else
    output_asm_insn ("%!jmp\t%A0", &call_op);
  return (ix86_harden_sls & harden_sls_indirect_jmp) ? "int3" : "";
}

/* Output return instrumentation for current function if needed.  */

static void
output_return_instrumentation (void)
{
  if (ix86_instrument_return != instrument_return_none
      && flag_fentry
      && !DECL_NO_INSTRUMENT_FUNCTION_ENTRY_EXIT (cfun->decl))
    {
      if (ix86_flag_record_return)
	fprintf (asm_out_file, "1:\n");
      switch (ix86_instrument_return)
	{
	case instrument_return_call:
	  fprintf (asm_out_file, "\tcall\t__return__\n");
	  break;
	case instrument_return_nop5:
	  /* 5 byte nop: nopl 0(%[re]ax,%[re]ax,1)  */
	  fprintf (asm_out_file, ASM_BYTE "0x0f, 0x1f, 0x44, 0x00, 0x00\n");
	  break;
	case instrument_return_none:
	  break;
	}

      if (ix86_flag_record_return)
	{
	  fprintf (asm_out_file, "\t.section __return_loc, \"a\",@progbits\n");
	  fprintf (asm_out_file, "\t.%s 1b\n", TARGET_64BIT ? "quad" : "long");
	  fprintf (asm_out_file, "\t.previous\n");
	}
    }
}

/* Output function return.  CALL_OP is the jump target.  Add a REP
   prefix to RET if LONG_P is true and function return is kept.  */

const char *
ix86_output_function_return (bool long_p)
{
  output_return_instrumentation ();

  if (cfun->machine->function_return_type != indirect_branch_keep)
    {
      char thunk_name[32];
      enum indirect_thunk_prefix need_prefix
	= indirect_thunk_need_prefix (current_output_insn);

      if (cfun->machine->function_return_type
	  != indirect_branch_thunk_inline)
	{
	  bool need_thunk = (cfun->machine->function_return_type
			     == indirect_branch_thunk);
	  indirect_thunk_name (thunk_name, INVALID_REGNUM, need_prefix,
			       true);
	  indirect_return_needed |= need_thunk;
	  fprintf (asm_out_file, "\tjmp\t");
	  assemble_name (asm_out_file, thunk_name);
	  putc ('\n', asm_out_file);
	}
      else
	output_indirect_thunk (INVALID_REGNUM);

      return "";
    }

  output_asm_insn (long_p ? "rep%; ret" : "ret", nullptr);
  return (ix86_harden_sls & harden_sls_return) ? "int3" : "";
}

/* Output indirect function return.  RET_OP is the function return
   target.  */

const char *
ix86_output_indirect_function_return (rtx ret_op)
{
  if (cfun->machine->function_return_type != indirect_branch_keep)
    {
      char thunk_name[32];
      enum indirect_thunk_prefix need_prefix
	= indirect_thunk_need_prefix (current_output_insn);
      unsigned int regno = REGNO (ret_op);
      gcc_assert (regno == CX_REG);

      if (cfun->machine->function_return_type
	  != indirect_branch_thunk_inline)
	{
	  bool need_thunk = (cfun->machine->function_return_type
			     == indirect_branch_thunk);
	  indirect_thunk_name (thunk_name, regno, need_prefix, true);

	  if (need_thunk)
	    {
	      indirect_return_via_cx = true;
	      SET_HARD_REG_BIT (indirect_thunks_used, CX_REG);
	    }
	  fprintf (asm_out_file, "\tjmp\t");
	  assemble_name (asm_out_file, thunk_name);
	  putc ('\n', asm_out_file);
	}
      else
	output_indirect_thunk (regno);
    }
  else
    {
      output_asm_insn ("%!jmp\t%A0", &ret_op);
      if (ix86_harden_sls & harden_sls_indirect_jmp)
	fputs ("\tint3\n", asm_out_file);
    }
  return "";
}

/* Output the assembly for a call instruction.  */

const char *
ix86_output_call_insn (rtx_insn *insn, rtx call_op)
{
  bool direct_p = constant_call_address_operand (call_op, VOIDmode);
  bool output_indirect_p
    = (!TARGET_SEH
       && cfun->machine->indirect_branch_type != indirect_branch_keep);
  bool seh_nop_p = false;
  const char *xasm;

  if (SIBLING_CALL_P (insn))
    {
      output_return_instrumentation ();
      if (direct_p)
	{
	  if (ix86_nopic_noplt_attribute_p (call_op))
	    {
	      direct_p = false;
	      if (TARGET_64BIT)
		{
		  if (output_indirect_p)
		    xasm = "{%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
		  else
		    xasm = "%!jmp\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
		}
	      else
		{
		  if (output_indirect_p)
		    xasm = "{%p0@GOT|[DWORD PTR %p0@GOT]}";
		  else
		    xasm = "%!jmp\t{*%p0@GOT|[DWORD PTR %p0@GOT]}";
		}
	    }
	  else
	    xasm = "%!jmp\t%P0";
	}
      /* SEH epilogue detection requires the indirect branch case
	 to include REX.W.  */
      else if (TARGET_SEH)
	xasm = "%!rex.W jmp\t%A0";
      else
	{
	  if (output_indirect_p)
	    xasm = "%0";
	  else
	    xasm = "%!jmp\t%A0";
	}

      if (output_indirect_p && !direct_p)
	ix86_output_indirect_branch (call_op, xasm, true);
      else
	{
	  output_asm_insn (xasm, &call_op);
	  if (!direct_p
	      && (ix86_harden_sls & harden_sls_indirect_jmp))
	    return "int3";
	}
      return "";
    }

  /* SEH unwinding can require an extra nop to be emitted in several
     circumstances.  Determine if we have one of those.  */
  if (TARGET_SEH)
    {
      rtx_insn *i;

      for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i))
	{
	  /* Prevent a catch region from being adjacent to a jump that would
	     be interpreted as an epilogue sequence by the unwinder.  */
	  if (JUMP_P(i) && CROSSING_JUMP_P (i))
	    {
	      seh_nop_p = true;
	      break;
	    }
	    
	  /* If we get to another real insn, we don't need the nop.  */
	  if (INSN_P (i))
	    break;

	  /* If we get to the epilogue note, prevent a catch region from
	     being adjacent to the standard epilogue sequence.  Note that,
	     if non-call exceptions are enabled, we already did it during
	     epilogue expansion, or else, if the insn can throw internally,
	     we already did it during the reorg pass.  */
	  if (NOTE_P (i) && NOTE_KIND (i) == NOTE_INSN_EPILOGUE_BEG
	      && !flag_non_call_exceptions
	      && !can_throw_internal (insn))
	    {
	      seh_nop_p = true;
	      break;
	    }
	}

      /* If we didn't find a real insn following the call, prevent the
	 unwinder from looking into the next function.  */
      if (i == NULL)
	seh_nop_p = true;
    }

  if (direct_p)
    {
      if (ix86_nopic_noplt_attribute_p (call_op))
	{
	  direct_p = false;
	  if (TARGET_64BIT)
	    {
	      if (output_indirect_p)
		xasm = "{%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
	      else
		xasm = "%!call\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
	    }
	  else
	    {
	      if (output_indirect_p)
		xasm = "{%p0@GOT|[DWORD PTR %p0@GOT]}";
	      else
		xasm = "%!call\t{*%p0@GOT|[DWORD PTR %p0@GOT]}";
	    }
	}
      else
	xasm = "%!call\t%P0";
    }
  else
    {
      if (output_indirect_p)
	xasm = "%0";
      else
	xasm = "%!call\t%A0";
    }

  if (output_indirect_p && !direct_p)
    ix86_output_indirect_branch (call_op, xasm, false);
  else
    output_asm_insn (xasm, &call_op);

  if (seh_nop_p)
    return "nop";

  return "";
}

/* Return a MEM corresponding to a stack slot with mode MODE.
   Allocate a new slot if necessary.

   The RTL for a function can have several slots available: N is
   which slot to use.  */

rtx
assign_386_stack_local (machine_mode mode, enum ix86_stack_slot n)
{
  struct stack_local_entry *s;

  gcc_assert (n < MAX_386_STACK_LOCALS);

  for (s = ix86_stack_locals; s; s = s->next)
    if (s->mode == mode && s->n == n)
      return validize_mem (copy_rtx (s->rtl));

  int align = 0;
  /* For DImode with SLOT_FLOATxFDI_387 use 32-bit
     alignment with -m32 -mpreferred-stack-boundary=2.  */
  if (mode == DImode
      && !TARGET_64BIT
      && n == SLOT_FLOATxFDI_387
      && ix86_preferred_stack_boundary < GET_MODE_ALIGNMENT (DImode))
    align = 32;
  s = ggc_alloc<stack_local_entry> ();
  s->n = n;
  s->mode = mode;
  s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), align);

  s->next = ix86_stack_locals;
  ix86_stack_locals = s;
  return validize_mem (copy_rtx (s->rtl));
}

static void
ix86_instantiate_decls (void)
{
  struct stack_local_entry *s;

  for (s = ix86_stack_locals; s; s = s->next)
    if (s->rtl != NULL_RTX)
      instantiate_decl_rtl (s->rtl);
}

/* Check whether x86 address PARTS is a pc-relative address.  */

bool
ix86_rip_relative_addr_p (struct ix86_address *parts)
{
  rtx base, index, disp;

  base = parts->base;
  index = parts->index;
  disp = parts->disp;

  if (disp && !base && !index)
    {
      if (TARGET_64BIT)
	{
	  rtx symbol = disp;

	  if (GET_CODE (disp) == CONST)
	    symbol = XEXP (disp, 0);
	  if (GET_CODE (symbol) == PLUS
	      && CONST_INT_P (XEXP (symbol, 1)))
	    symbol = XEXP (symbol, 0);

	  if (GET_CODE (symbol) == LABEL_REF
	      || (GET_CODE (symbol) == SYMBOL_REF
		  && SYMBOL_REF_TLS_MODEL (symbol) == 0)
	      || (GET_CODE (symbol) == UNSPEC
		  && (XINT (symbol, 1) == UNSPEC_GOTPCREL
		      || XINT (symbol, 1) == UNSPEC_PCREL
		      || XINT (symbol, 1) == UNSPEC_GOTNTPOFF)))
	    return true;
	}
    }
  return false;
}

/* Calculate the length of the memory address in the instruction encoding.
   Includes addr32 prefix, does not include the one-byte modrm, opcode,
   or other prefixes.  We never generate addr32 prefix for LEA insn.  */

int
memory_address_length (rtx addr, bool lea)
{
  struct ix86_address parts;
  rtx base, index, disp;
  int len;
  int ok;

  if (GET_CODE (addr) == PRE_DEC
      || GET_CODE (addr) == POST_INC
      || GET_CODE (addr) == PRE_MODIFY
      || GET_CODE (addr) == POST_MODIFY)
    return 0;

  ok = ix86_decompose_address (addr, &parts);
  gcc_assert (ok);

  len = (parts.seg == ADDR_SPACE_GENERIC) ? 0 : 1;

  /*  If this is not LEA instruction, add the length of addr32 prefix.  */
  if (TARGET_64BIT && !lea
      && (SImode_address_operand (addr, VOIDmode)
	  || (parts.base && GET_MODE (parts.base) == SImode)
	  || (parts.index && GET_MODE (parts.index) == SImode)))
    len++;

  base = parts.base;
  index = parts.index;
  disp = parts.disp;

  if (base && SUBREG_P (base))
    base = SUBREG_REG (base);
  if (index && SUBREG_P (index))
    index = SUBREG_REG (index);

  gcc_assert (base == NULL_RTX || REG_P (base));
  gcc_assert (index == NULL_RTX || REG_P (index));

  /* Rule of thumb:
       - esp as the base always wants an index,
       - ebp as the base always wants a displacement,
       - r12 as the base always wants an index,
       - r13 as the base always wants a displacement.  */

  /* Register Indirect.  */
  if (base && !index && !disp)
    {
      /* esp (for its index) and ebp (for its displacement) need
	 the two-byte modrm form.  Similarly for r12 and r13 in 64-bit
	 code.  */
      if (base == arg_pointer_rtx
	  || base == frame_pointer_rtx
	  || REGNO (base) == SP_REG
	  || REGNO (base) == BP_REG
	  || REGNO (base) == R12_REG
	  || REGNO (base) == R13_REG)
	len++;
    }

  /* Direct Addressing.  In 64-bit mode mod 00 r/m 5
     is not disp32, but disp32(%rip), so for disp32
     SIB byte is needed, unless print_operand_address
     optimizes it into disp32(%rip) or (%rip) is implied
     by UNSPEC.  */
  else if (disp && !base && !index)
    {
      len += 4;
      if (!ix86_rip_relative_addr_p (&parts))
	len++;
    }
  else
    {
      /* Find the length of the displacement constant.  */
      if (disp)
	{
	  if (base && satisfies_constraint_K (disp))
	    len += 1;
	  else
	    len += 4;
	}
      /* ebp always wants a displacement.  Similarly r13.  */
      else if (base && (REGNO (base) == BP_REG || REGNO (base) == R13_REG))
	len++;

      /* An index requires the two-byte modrm form....  */
      if (index
	  /* ...like esp (or r12), which always wants an index.  */
	  || base == arg_pointer_rtx
	  || base == frame_pointer_rtx
	  || (base && (REGNO (base) == SP_REG || REGNO (base) == R12_REG)))
	len++;
    }

  return len;
}

/* Compute default value for "length_immediate" attribute.  When SHORTFORM
   is set, expect that insn have 8bit immediate alternative.  */
int
ix86_attr_length_immediate_default (rtx_insn *insn, bool shortform)
{
  int len = 0;
  int i;
  extract_insn_cached (insn);
  for (i = recog_data.n_operands - 1; i >= 0; --i)
    if (CONSTANT_P (recog_data.operand[i]))
      {
        enum attr_mode mode = get_attr_mode (insn);

	gcc_assert (!len);
	if (shortform && CONST_INT_P (recog_data.operand[i]))
	  {
	    HOST_WIDE_INT ival = INTVAL (recog_data.operand[i]);
	    switch (mode)
	      {
	      case MODE_QI:
		len = 1;
		continue;
	      case MODE_HI:
		ival = trunc_int_for_mode (ival, HImode);
		break;
	      case MODE_SI:
		ival = trunc_int_for_mode (ival, SImode);
		break;
	      default:
		break;
	      }
	    if (IN_RANGE (ival, -128, 127))
	      {
		len = 1;
		continue;
	      }
	  }
	switch (mode)
	  {
	  case MODE_QI:
	    len = 1;
	    break;
	  case MODE_HI:
	    len = 2;
	    break;
	  case MODE_SI:
	    len = 4;
	    break;
	  /* Immediates for DImode instructions are encoded
	     as 32bit sign extended values.  */
	  case MODE_DI:
	    len = 4;
	    break;
	  default:
	    fatal_insn ("unknown insn mode", insn);
	}
      }
  return len;
}

/* Compute default value for "length_address" attribute.  */
int
ix86_attr_length_address_default (rtx_insn *insn)
{
  int i;

  if (get_attr_type (insn) == TYPE_LEA)
    {
      rtx set = PATTERN (insn), addr;

      if (GET_CODE (set) == PARALLEL)
	set = XVECEXP (set, 0, 0);

      gcc_assert (GET_CODE (set) == SET);

      addr = SET_SRC (set);

      return memory_address_length (addr, true);
    }

  extract_insn_cached (insn);
  for (i = recog_data.n_operands - 1; i >= 0; --i)
    {
      rtx op = recog_data.operand[i];
      if (MEM_P (op))
	{
	  constrain_operands_cached (insn, reload_completed);
	  if (which_alternative != -1)
	    {
	      const char *constraints = recog_data.constraints[i];
	      int alt = which_alternative;

	      while (*constraints == '=' || *constraints == '+')
		constraints++;
	      while (alt-- > 0)
	        while (*constraints++ != ',')
		  ;
	      /* Skip ignored operands.  */
	      if (*constraints == 'X')
		continue;
	    }

	  int len = memory_address_length (XEXP (op, 0), false);

	  /* Account for segment prefix for non-default addr spaces.  */
	  if (!ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (op)))
	    len++;

	  return len;
	}
    }
  return 0;
}

/* Compute default value for "length_vex" attribute. It includes
   2 or 3 byte VEX prefix and 1 opcode byte.  */

int
ix86_attr_length_vex_default (rtx_insn *insn, bool has_0f_opcode,
			      bool has_vex_w)
{
  int i, reg_only = 2 + 1;
  bool has_mem = false;

  /* Only 0f opcode can use 2 byte VEX prefix and  VEX W bit uses 3
     byte VEX prefix.  */
  if (!has_0f_opcode || has_vex_w)
    return 3 + 1;

 /* We can always use 2 byte VEX prefix in 32bit.  */
  if (!TARGET_64BIT)
    return 2 + 1;

  extract_insn_cached (insn);

  for (i = recog_data.n_operands - 1; i >= 0; --i)
    if (REG_P (recog_data.operand[i]))
      {
	/* REX.W bit uses 3 byte VEX prefix.
	   REX2 with vex use extended EVEX prefix length is 4-byte.  */
	if (GET_MODE (recog_data.operand[i]) == DImode
	    && GENERAL_REG_P (recog_data.operand[i]))
	  return 3 + 1;

	/* REX.B bit requires 3-byte VEX. Right here we don't know which
	   operand will be encoded using VEX.B, so be conservative.
	   REX2 with vex use extended EVEX prefix length is 4-byte.  */
	if (REX_INT_REGNO_P (recog_data.operand[i])
	    || REX2_INT_REGNO_P (recog_data.operand[i])
	    || REX_SSE_REGNO_P (recog_data.operand[i]))
	  reg_only = 3 + 1;
      }
    else if (MEM_P (recog_data.operand[i]))
      {
	/* REX2.X or REX2.B bits use 3 byte VEX prefix.  */
	if (x86_extended_rex2reg_mentioned_p (recog_data.operand[i]))
	  return 4;

	/* REX.X or REX.B bits use 3 byte VEX prefix.  */
	if (x86_extended_reg_mentioned_p (recog_data.operand[i]))
	  return 3 + 1;

	has_mem = true;
      }

  return has_mem ? 2 + 1 : reg_only;
}


static bool
ix86_class_likely_spilled_p (reg_class_t);

/* Returns true if lhs of insn is HW function argument register and set up
   is_spilled to true if it is likely spilled HW register.  */
static bool
insn_is_function_arg (rtx insn, bool* is_spilled)
{
  rtx dst;

  if (!NONDEBUG_INSN_P (insn))
    return false;
  /* Call instructions are not movable, ignore it.  */
  if (CALL_P (insn))
    return false;
  insn = PATTERN (insn);
  if (GET_CODE (insn) == PARALLEL)
    insn = XVECEXP (insn, 0, 0);
  if (GET_CODE (insn) != SET)
    return false;
  dst = SET_DEST (insn);
  if (REG_P (dst) && HARD_REGISTER_P (dst)
      && ix86_function_arg_regno_p (REGNO (dst)))
    {
      /* Is it likely spilled HW register?  */
      if (!TEST_HARD_REG_BIT (fixed_reg_set, REGNO (dst))
	  && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (dst))))
	*is_spilled = true;
      return true;
    }
  return false;
}

/* Add output dependencies for chain of function adjacent arguments if only
   there is a move to likely spilled HW register.  Return first argument
   if at least one dependence was added or NULL otherwise.  */
static rtx_insn *
add_parameter_dependencies (rtx_insn *call, rtx_insn *head)
{
  rtx_insn *insn;
  rtx_insn *last = call;
  rtx_insn *first_arg = NULL;
  bool is_spilled = false;

  head = PREV_INSN (head);

  /* Find nearest to call argument passing instruction.  */
  while (true)
    {
      last = PREV_INSN (last);
      if (last == head)
	return NULL;
      if (!NONDEBUG_INSN_P (last))
	continue;
      if (insn_is_function_arg (last, &is_spilled))
	break;
      return NULL;
    }

  first_arg = last;
  while (true)
    {
      insn = PREV_INSN (last);
      if (!INSN_P (insn))
	break;
      if (insn == head)
	break;
      if (!NONDEBUG_INSN_P (insn))
	{
	  last = insn;
	  continue;
	}
      if (insn_is_function_arg (insn, &is_spilled))
	{
	  /* Add output depdendence between two function arguments if chain
	     of output arguments contains likely spilled HW registers.  */
	  if (is_spilled)
	    add_dependence (first_arg, insn, REG_DEP_OUTPUT);
	  first_arg = last = insn;
	}
      else
	break;
    }
  if (!is_spilled)
    return NULL;
  return first_arg;
}

/* Add output or anti dependency from insn to first_arg to restrict its code
   motion.  */
static void
avoid_func_arg_motion (rtx_insn *first_arg, rtx_insn *insn)
{
  rtx set;
  rtx tmp;

  set = single_set (insn);
  if (!set)
    return;
  tmp = SET_DEST (set);
  if (REG_P (tmp))
    {
      /* Add output dependency to the first function argument.  */
      add_dependence (first_arg, insn, REG_DEP_OUTPUT);
      return;
    }
  /* Add anti dependency.  */
  add_dependence (first_arg, insn, REG_DEP_ANTI);
}

/* Avoid cross block motion of function argument through adding dependency
   from the first non-jump instruction in bb.  */
static void
add_dependee_for_func_arg (rtx_insn *arg, basic_block bb)
{
  rtx_insn *insn = BB_END (bb);

  while (insn)
    {
      if (NONDEBUG_INSN_P (insn) && NONJUMP_INSN_P (insn))
	{
	  rtx set = single_set (insn);
	  if (set)
	    {
	      avoid_func_arg_motion (arg, insn);
	      return;
	    }
	}
      if (insn == BB_HEAD (bb))
	return;
      insn = PREV_INSN (insn);
    }
}

/* Hook for pre-reload schedule - avoid motion of function arguments
   passed in likely spilled HW registers.  */
static void
ix86_dependencies_evaluation_hook (rtx_insn *head, rtx_insn *tail)
{
  rtx_insn *insn;
  rtx_insn *first_arg = NULL;
  if (reload_completed)
    return;
  while (head != tail && DEBUG_INSN_P (head))
    head = NEXT_INSN (head);
  for (insn = tail; insn != head; insn = PREV_INSN (insn))
    if (INSN_P (insn) && CALL_P (insn))
      {
	first_arg = add_parameter_dependencies (insn, head);
	if (first_arg)
	  {
	    /* Add dependee for first argument to predecessors if only
	       region contains more than one block.  */
	    basic_block bb =  BLOCK_FOR_INSN (insn);
	    int rgn = CONTAINING_RGN (bb->index);
	    int nr_blks = RGN_NR_BLOCKS (rgn);
	    /* Skip trivial regions and region head blocks that can have
	       predecessors outside of region.  */
	    if (nr_blks > 1 && BLOCK_TO_BB (bb->index) != 0)
	      {
		edge e;
		edge_iterator ei;

		/* Regions are SCCs with the exception of selective
		   scheduling with pipelining of outer blocks enabled.
		   So also check that immediate predecessors of a non-head
		   block are in the same region.  */
		FOR_EACH_EDGE (e, ei, bb->preds)
		  {
		    /* Avoid creating of loop-carried dependencies through
		       using topological ordering in the region.  */
		    if (rgn == CONTAINING_RGN (e->src->index)
			&& BLOCK_TO_BB (bb->index) > BLOCK_TO_BB (e->src->index))
		      add_dependee_for_func_arg (first_arg, e->src); 
		  }
	      }
	    insn = first_arg;
	    if (insn == head)
	      break;
	  }
      }
    else if (first_arg)
      avoid_func_arg_motion (first_arg, insn);
}

/* Hook for pre-reload schedule - set priority of moves from likely spilled
   HW registers to maximum, to schedule them at soon as possible. These are
   moves from function argument registers at the top of the function entry
   and moves from function return value registers after call.  */
static int
ix86_adjust_priority (rtx_insn *insn, int priority)
{
  rtx set;

  if (reload_completed)
    return priority;

  if (!NONDEBUG_INSN_P (insn))
    return priority;

  set = single_set (insn);
  if (set)
    {
      rtx tmp = SET_SRC (set);
      if (REG_P (tmp)
          && HARD_REGISTER_P (tmp)
          && !TEST_HARD_REG_BIT (fixed_reg_set, REGNO (tmp))
          && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (tmp))))
	return current_sched_info->sched_max_insns_priority;
    }

  return priority;
}

/* Prepare for scheduling pass.  */
static void
ix86_sched_init_global (FILE *, int, int)
{
  /* Install scheduling hooks for current CPU.  Some of these hooks are used
     in time-critical parts of the scheduler, so we only set them up when
     they are actually used.  */
  switch (ix86_tune)
    {
    case PROCESSOR_CORE2:
    case PROCESSOR_NEHALEM:
    case PROCESSOR_SANDYBRIDGE:
    case PROCESSOR_HASWELL:
    case PROCESSOR_TREMONT:
    case PROCESSOR_ALDERLAKE:
    case PROCESSOR_GENERIC:
      /* Do not perform multipass scheduling for pre-reload schedule
         to save compile time.  */
      if (reload_completed)
	{
	  ix86_core2i7_init_hooks ();
	  break;
	}
      /* Fall through.  */
    default:
      targetm.sched.dfa_post_advance_cycle = NULL;
      targetm.sched.first_cycle_multipass_init = NULL;
      targetm.sched.first_cycle_multipass_begin = NULL;
      targetm.sched.first_cycle_multipass_issue = NULL;
      targetm.sched.first_cycle_multipass_backtrack = NULL;
      targetm.sched.first_cycle_multipass_end = NULL;
      targetm.sched.first_cycle_multipass_fini = NULL;
      break;
    }
}


/* Implement TARGET_STATIC_RTX_ALIGNMENT.  */

static HOST_WIDE_INT
ix86_static_rtx_alignment (machine_mode mode)
{
  if (mode == DFmode)
    return 64;
  if (ALIGN_MODE_128 (mode))
    return MAX (128, GET_MODE_ALIGNMENT (mode));
  return GET_MODE_ALIGNMENT (mode);
}

/* Implement TARGET_CONSTANT_ALIGNMENT.  */

static HOST_WIDE_INT
ix86_constant_alignment (const_tree exp, HOST_WIDE_INT align)
{
  if (TREE_CODE (exp) == REAL_CST || TREE_CODE (exp) == VECTOR_CST
      || TREE_CODE (exp) == INTEGER_CST)
    {
      machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
      HOST_WIDE_INT mode_align = ix86_static_rtx_alignment (mode);
      return MAX (mode_align, align);
    }
  else if (!optimize_size && TREE_CODE (exp) == STRING_CST
	   && TREE_STRING_LENGTH (exp) >= 31 && align < BITS_PER_WORD)
    return BITS_PER_WORD;

  return align;
}

/* Implement TARGET_EMPTY_RECORD_P.  */

static bool
ix86_is_empty_record (const_tree type)
{
  if (!TARGET_64BIT)
    return false;
  return default_is_empty_record (type);
}

/* Implement TARGET_WARN_PARAMETER_PASSING_ABI.  */

static void
ix86_warn_parameter_passing_abi (cumulative_args_t cum_v, tree type)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);

  if (!cum->warn_empty)
    return;

  if (!TYPE_EMPTY_P (type))
    return;

  /* Don't warn if the function isn't visible outside of the TU.  */
  if (cum->decl && !TREE_PUBLIC (cum->decl))
    return;

  const_tree ctx = get_ultimate_context (cum->decl);
  if (ctx != NULL_TREE
      && !TRANSLATION_UNIT_WARN_EMPTY_P (ctx))
    return;

  /* If the actual size of the type is zero, then there is no change
     in how objects of this size are passed.  */
  if (int_size_in_bytes (type) == 0)
    return;

  warning (OPT_Wabi, "empty class %qT parameter passing ABI "
	   "changes in %<-fabi-version=12%> (GCC 8)", type);

  /* Only warn once.  */
  cum->warn_empty = false;
}

/* This hook returns name of multilib ABI.  */

static const char *
ix86_get_multilib_abi_name (void)
{
  if (!(TARGET_64BIT_P (ix86_isa_flags)))
    return "i386";
  else if (TARGET_X32_P (ix86_isa_flags))
    return "x32";
  else
    return "x86_64";
}

/* Compute the alignment for a variable for Intel MCU psABI.  TYPE is
   the data type, and ALIGN is the alignment that the object would
   ordinarily have.  */

static int
iamcu_alignment (tree type, int align)
{
  machine_mode mode;

  if (align < 32 || TYPE_USER_ALIGN (type))
    return align;

  /* Intel MCU psABI specifies scalar types > 4 bytes aligned to 4
     bytes.  */
  type = strip_array_types (type);
  if (TYPE_ATOMIC (type))
    return align;

  mode = TYPE_MODE (type);
  switch (GET_MODE_CLASS (mode))
    {
    case MODE_INT:
    case MODE_COMPLEX_INT:
    case MODE_COMPLEX_FLOAT:
    case MODE_FLOAT:
    case MODE_DECIMAL_FLOAT:
      return 32;
    default:
      return align;
    }
}

/* Compute the alignment for a static variable.
   TYPE is the data type, and ALIGN is the alignment that
   the object would ordinarily have.  The value of this function is used
   instead of that alignment to align the object.  */

int
ix86_data_alignment (tree type, unsigned int align, bool opt)
{
  /* GCC 4.8 and earlier used to incorrectly assume this alignment even
     for symbols from other compilation units or symbols that don't need
     to bind locally.  In order to preserve some ABI compatibility with
     those compilers, ensure we don't decrease alignment from what we
     used to assume.  */

  unsigned int max_align_compat = MIN (256, MAX_OFILE_ALIGNMENT);

  /* A data structure, equal or greater than the size of a cache line
     (64 bytes in the Pentium 4 and other recent Intel processors, including
     processors based on Intel Core microarchitecture) should be aligned
     so that its base address is a multiple of a cache line size.  */

  unsigned int max_align
    = MIN ((unsigned) ix86_tune_cost->prefetch_block * 8, MAX_OFILE_ALIGNMENT);

  if (max_align < BITS_PER_WORD)
    max_align = BITS_PER_WORD;

  switch (ix86_align_data_type)
    {
    case ix86_align_data_type_abi: opt = false; break;
    case ix86_align_data_type_compat: max_align = BITS_PER_WORD; break;
    case ix86_align_data_type_cacheline: break;
    }

  if (TARGET_IAMCU)
    align = iamcu_alignment (type, align);

  if (opt
      && AGGREGATE_TYPE_P (type)
      && TYPE_SIZE (type)
      && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST)
    {
      if (wi::geu_p (wi::to_wide (TYPE_SIZE (type)), max_align_compat)
	  && align < max_align_compat)
	align = max_align_compat;
      if (wi::geu_p (wi::to_wide (TYPE_SIZE (type)), max_align)
	  && align < max_align)
	align = max_align;
    }

  /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
     to 16byte boundary.  */
  if (TARGET_64BIT)
    {
      if ((opt ? AGGREGATE_TYPE_P (type) : TREE_CODE (type) == ARRAY_TYPE)
	  && TYPE_SIZE (type)
	  && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
	  && wi::geu_p (wi::to_wide (TYPE_SIZE (type)), 128)
	  && align < 128)
	return 128;
    }

  if (!opt)
    return align;

  if (TREE_CODE (type) == ARRAY_TYPE)
    {
      if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
	return 64;
      if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
	return 128;
    }
  else if (TREE_CODE (type) == COMPLEX_TYPE)
    {

      if (TYPE_MODE (type) == DCmode && align < 64)
	return 64;
      if ((TYPE_MODE (type) == XCmode
	   || TYPE_MODE (type) == TCmode) && align < 128)
	return 128;
    }
  else if (RECORD_OR_UNION_TYPE_P (type)
	   && TYPE_FIELDS (type))
    {
      if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
	return 64;
      if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
	return 128;
    }
  else if (SCALAR_FLOAT_TYPE_P (type) || VECTOR_TYPE_P (type)
	   || TREE_CODE (type) == INTEGER_TYPE)
    {
      if (TYPE_MODE (type) == DFmode && align < 64)
	return 64;
      if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
	return 128;
    }

  return align;
}

/* Implememnt TARGET_LOWER_LOCAL_DECL_ALIGNMENT.  */
static void
ix86_lower_local_decl_alignment (tree decl)
{
  unsigned int new_align = ix86_local_alignment (decl, VOIDmode,
						 DECL_ALIGN (decl), true);
  if (new_align < DECL_ALIGN (decl))
    SET_DECL_ALIGN (decl, new_align);
}

/* Compute the alignment for a local variable or a stack slot.  EXP is
   the data type or decl itself, MODE is the widest mode available and
   ALIGN is the alignment that the object would ordinarily have.  The
   value of this macro is used instead of that alignment to align the
   object.  */

unsigned int
ix86_local_alignment (tree exp, machine_mode mode,
		      unsigned int align, bool may_lower)
{
  tree type, decl;

  if (exp && DECL_P (exp))
    {
      type = TREE_TYPE (exp);
      decl = exp;
    }
  else
    {
      type = exp;
      decl = NULL;
    }

  /* Don't do dynamic stack realignment for long long objects with
     -mpreferred-stack-boundary=2.  */
  if (may_lower
      && !TARGET_64BIT
      && align == 64
      && ix86_preferred_stack_boundary < 64
      && (mode == DImode || (type && TYPE_MODE (type) == DImode))
      && (!type || (!TYPE_USER_ALIGN (type)
		    && !TYPE_ATOMIC (strip_array_types (type))))
      && (!decl || !DECL_USER_ALIGN (decl)))
    align = 32;

  /* If TYPE is NULL, we are allocating a stack slot for caller-save
     register in MODE.  We will return the largest alignment of XF
     and DF.  */
  if (!type)
    {
      if (mode == XFmode && align < GET_MODE_ALIGNMENT (DFmode))
	align = GET_MODE_ALIGNMENT (DFmode);
      return align;
    }

  /* Don't increase alignment for Intel MCU psABI.  */
  if (TARGET_IAMCU)
    return align;

  /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
     to 16byte boundary.  Exact wording is:

     An array uses the same alignment as its elements, except that a local or
     global array variable of length at least 16 bytes or
     a C99 variable-length array variable always has alignment of at least 16 bytes.

     This was added to allow use of aligned SSE instructions at arrays.  This
     rule is meant for static storage (where compiler cannot do the analysis
     by itself).  We follow it for automatic variables only when convenient.
     We fully control everything in the function compiled and functions from
     other unit cannot rely on the alignment.

     Exclude va_list type.  It is the common case of local array where
     we cannot benefit from the alignment.  

     TODO: Probably one should optimize for size only when var is not escaping.  */
  if (TARGET_64BIT && optimize_function_for_speed_p (cfun)
      && TARGET_SSE)
    {
      if (AGGREGATE_TYPE_P (type)
	  && (va_list_type_node == NULL_TREE
	      || (TYPE_MAIN_VARIANT (type)
		  != TYPE_MAIN_VARIANT (va_list_type_node)))
	  && TYPE_SIZE (type)
	  && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
	  && wi::geu_p (wi::to_wide (TYPE_SIZE (type)), 128)
	  && align < 128)
	return 128;
    }
  if (TREE_CODE (type) == ARRAY_TYPE)
    {
      if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
	return 64;
      if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
	return 128;
    }
  else if (TREE_CODE (type) == COMPLEX_TYPE)
    {
      if (TYPE_MODE (type) == DCmode && align < 64)
	return 64;
      if ((TYPE_MODE (type) == XCmode
	   || TYPE_MODE (type) == TCmode) && align < 128)
	return 128;
    }
  else if (RECORD_OR_UNION_TYPE_P (type)
	   && TYPE_FIELDS (type))
    {
      if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
	return 64;
      if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
	return 128;
    }
  else if (SCALAR_FLOAT_TYPE_P (type) || VECTOR_TYPE_P (type)
	   || TREE_CODE (type) == INTEGER_TYPE)
    {

      if (TYPE_MODE (type) == DFmode && align < 64)
	return 64;
      if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
	return 128;
    }
  return align;
}

/* Compute the minimum required alignment for dynamic stack realignment
   purposes for a local variable, parameter or a stack slot.  EXP is
   the data type or decl itself, MODE is its mode and ALIGN is the
   alignment that the object would ordinarily have.  */

unsigned int
ix86_minimum_alignment (tree exp, machine_mode mode,
			unsigned int align)
{
  tree type, decl;

  if (exp && DECL_P (exp))
    {
      type = TREE_TYPE (exp);
      decl = exp;
    }
  else
    {
      type = exp;
      decl = NULL;
    }

  if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64)
    return align;

  /* Don't do dynamic stack realignment for long long objects with
     -mpreferred-stack-boundary=2.  */
  if ((mode == DImode || (type && TYPE_MODE (type) == DImode))
      && (!type || (!TYPE_USER_ALIGN (type)
		    && !TYPE_ATOMIC (strip_array_types (type))))
      && (!decl || !DECL_USER_ALIGN (decl)))
    {
      gcc_checking_assert (!TARGET_STV);
      return 32;
    }

  return align;
}

/* Find a location for the static chain incoming to a nested function.
   This is a register, unless all free registers are used by arguments.  */

static rtx
ix86_static_chain (const_tree fndecl_or_type, bool incoming_p)
{
  unsigned regno;

  if (TARGET_64BIT)
    {
      /* We always use R10 in 64-bit mode.  */
      regno = R10_REG;
    }
  else
    {
      const_tree fntype, fndecl;
      unsigned int ccvt;

      /* By default in 32-bit mode we use ECX to pass the static chain.  */
      regno = CX_REG;

      if (TREE_CODE (fndecl_or_type) == FUNCTION_DECL)
	{
          fntype = TREE_TYPE (fndecl_or_type);
	  fndecl = fndecl_or_type;
	}
      else
	{
	  fntype = fndecl_or_type;
	  fndecl = NULL;
	}

      ccvt = ix86_get_callcvt (fntype);
      if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
	{
	  /* Fastcall functions use ecx/edx for arguments, which leaves
	     us with EAX for the static chain.
	     Thiscall functions use ecx for arguments, which also
	     leaves us with EAX for the static chain.  */
	  regno = AX_REG;
	}
      else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
	{
	  /* Thiscall functions use ecx for arguments, which leaves
	     us with EAX and EDX for the static chain.
	     We are using for abi-compatibility EAX.  */
	  regno = AX_REG;
	}
      else if (ix86_function_regparm (fntype, fndecl) == 3)
	{
	  /* For regparm 3, we have no free call-clobbered registers in
	     which to store the static chain.  In order to implement this,
	     we have the trampoline push the static chain to the stack.
	     However, we can't push a value below the return address when
	     we call the nested function directly, so we have to use an
	     alternate entry point.  For this we use ESI, and have the
	     alternate entry point push ESI, so that things appear the
	     same once we're executing the nested function.  */
	  if (incoming_p)
	    {
	      if (fndecl == current_function_decl
		  && !ix86_static_chain_on_stack)
		{
		  gcc_assert (!reload_completed);
		  ix86_static_chain_on_stack = true;
		}
	      return gen_frame_mem (SImode,
				    plus_constant (Pmode,
						   arg_pointer_rtx, -8));
	    }
	  regno = SI_REG;
	}
    }

  return gen_rtx_REG (Pmode, regno);
}

/* Emit RTL insns to initialize the variable parts of a trampoline.
   FNDECL is the decl of the target address; M_TRAMP is a MEM for
   the trampoline, and CHAIN_VALUE is an RTX for the static chain
   to be passed to the target function.  */

static void
ix86_trampoline_init (rtx m_tramp, tree fndecl, rtx chain_value)
{
  rtx mem, fnaddr;
  int opcode;
  int offset = 0;
  bool need_endbr = (flag_cf_protection & CF_BRANCH);

  fnaddr = XEXP (DECL_RTL (fndecl), 0);

  if (TARGET_64BIT)
    {
      int size;

      if (need_endbr)
	{
	  /* Insert ENDBR64.  */
	  mem = adjust_address (m_tramp, SImode, offset);
	  emit_move_insn (mem, gen_int_mode (0xfa1e0ff3, SImode));
	  offset += 4;
	}

      /* Load the function address to r11.  Try to load address using
	 the shorter movl instead of movabs.  We may want to support
	 movq for kernel mode, but kernel does not use trampolines at
	 the moment.  FNADDR is a 32bit address and may not be in
	 DImode when ptr_mode == SImode.  Always use movl in this
	 case.  */
      if (ptr_mode == SImode
	  || x86_64_zext_immediate_operand (fnaddr, VOIDmode))
	{
	  fnaddr = copy_addr_to_reg (fnaddr);

	  mem = adjust_address (m_tramp, HImode, offset);
	  emit_move_insn (mem, gen_int_mode (0xbb41, HImode));

	  mem = adjust_address (m_tramp, SImode, offset + 2);
	  emit_move_insn (mem, gen_lowpart (SImode, fnaddr));
	  offset += 6;
	}
      else
	{
	  mem = adjust_address (m_tramp, HImode, offset);
	  emit_move_insn (mem, gen_int_mode (0xbb49, HImode));

	  mem = adjust_address (m_tramp, DImode, offset + 2);
	  emit_move_insn (mem, fnaddr);
	  offset += 10;
	}

      /* Load static chain using movabs to r10.  Use the shorter movl
         instead of movabs when ptr_mode == SImode.  */
      if (ptr_mode == SImode)
	{
	  opcode = 0xba41;
	  size = 6;
	}
      else
	{
	  opcode = 0xba49;
	  size = 10;
	}

      mem = adjust_address (m_tramp, HImode, offset);
      emit_move_insn (mem, gen_int_mode (opcode, HImode));

      mem = adjust_address (m_tramp, ptr_mode, offset + 2);
      emit_move_insn (mem, chain_value);
      offset += size;

      /* Jump to r11; the last (unused) byte is a nop, only there to
	 pad the write out to a single 32-bit store.  */
      mem = adjust_address (m_tramp, SImode, offset);
      emit_move_insn (mem, gen_int_mode (0x90e3ff49, SImode));
      offset += 4;
    }
  else
    {
      rtx disp, chain;

      /* Depending on the static chain location, either load a register
	 with a constant, or push the constant to the stack.  All of the
	 instructions are the same size.  */
      chain = ix86_static_chain (fndecl, true);
      if (REG_P (chain))
	{
	  switch (REGNO (chain))
	    {
	    case AX_REG:
	      opcode = 0xb8; break;
	    case CX_REG:
	      opcode = 0xb9; break;
	    default:
	      gcc_unreachable ();
	    }
	}
      else
	opcode = 0x68;

      if (need_endbr)
	{
	  /* Insert ENDBR32.  */
	  mem = adjust_address (m_tramp, SImode, offset);
	  emit_move_insn (mem, gen_int_mode (0xfb1e0ff3, SImode));
	  offset += 4;
	}

      mem = adjust_address (m_tramp, QImode, offset);
      emit_move_insn (mem, gen_int_mode (opcode, QImode));

      mem = adjust_address (m_tramp, SImode, offset + 1);
      emit_move_insn (mem, chain_value);
      offset += 5;

      mem = adjust_address (m_tramp, QImode, offset);
      emit_move_insn (mem, gen_int_mode (0xe9, QImode));

      mem = adjust_address (m_tramp, SImode, offset + 1);

      /* Compute offset from the end of the jmp to the target function.
	 In the case in which the trampoline stores the static chain on
	 the stack, we need to skip the first insn which pushes the
	 (call-saved) register static chain; this push is 1 byte.  */
      offset += 5;
      int skip = MEM_P (chain) ? 1 : 0;
      /* Skip ENDBR32 at the entry of the target function.  */
      if (need_endbr
	  && !cgraph_node::get (fndecl)->only_called_directly_p ())
	skip += 4;
      disp = expand_binop (SImode, sub_optab, fnaddr,
			   plus_constant (Pmode, XEXP (m_tramp, 0),
					  offset - skip),
			   NULL_RTX, 1, OPTAB_DIRECT);
      emit_move_insn (mem, disp);
    }

  gcc_assert (offset <= TRAMPOLINE_SIZE);

#ifdef HAVE_ENABLE_EXECUTE_STACK
#ifdef CHECK_EXECUTE_STACK_ENABLED
  if (CHECK_EXECUTE_STACK_ENABLED)
#endif
  emit_library_call (gen_rtx_SYMBOL_REF (Pmode, "__enable_execute_stack"),
		     LCT_NORMAL, VOIDmode, XEXP (m_tramp, 0), Pmode);
#endif
}

static bool
ix86_allocate_stack_slots_for_args (void)
{
  /* Naked functions should not allocate stack slots for arguments.  */
  return !ix86_function_naked (current_function_decl);
}

static bool
ix86_warn_func_return (tree decl)
{
  /* Naked functions are implemented entirely in assembly, including the
     return sequence, so suppress warnings about this.  */
  return !ix86_function_naked (decl);
}

/* Return the shift count of a vector by scalar shift builtin second argument
   ARG1.  */
static tree
ix86_vector_shift_count (tree arg1)
{
  if (tree_fits_uhwi_p (arg1))
    return arg1;
  else if (TREE_CODE (arg1) == VECTOR_CST && CHAR_BIT == 8)
    {
      /* The count argument is weird, passed in as various 128-bit
	 (or 64-bit) vectors, the low 64 bits from it are the count.  */
      unsigned char buf[16];
      int len = native_encode_expr (arg1, buf, 16);
      if (len == 0)
	return NULL_TREE;
      tree t = native_interpret_expr (uint64_type_node, buf, len);
      if (t && tree_fits_uhwi_p (t))
	return t;
    }
  return NULL_TREE;
}

/* Return true if arg_mask is all ones, ELEMS is elements number of
   corresponding vector.  */
static bool
ix86_masked_all_ones (unsigned HOST_WIDE_INT elems, tree arg_mask)
{
  if (TREE_CODE (arg_mask) != INTEGER_CST)
    return false;

  unsigned HOST_WIDE_INT mask = TREE_INT_CST_LOW (arg_mask);
  if (elems == HOST_BITS_PER_WIDE_INT)
    return  mask == HOST_WIDE_INT_M1U;
  if ((mask | (HOST_WIDE_INT_M1U << elems)) != HOST_WIDE_INT_M1U)
    return false;

  return true;
}

static tree
ix86_fold_builtin (tree fndecl, int n_args,
		   tree *args, bool ignore ATTRIBUTE_UNUSED)
{
  if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
    {
      enum ix86_builtins fn_code
	= (enum ix86_builtins) DECL_MD_FUNCTION_CODE (fndecl);
      enum rtx_code rcode;
      bool is_vshift;
      unsigned HOST_WIDE_INT mask;

      switch (fn_code)
	{
	case IX86_BUILTIN_CPU_IS:
	case IX86_BUILTIN_CPU_SUPPORTS:
	  gcc_assert (n_args == 1);
	  return fold_builtin_cpu (fndecl, args);

	case IX86_BUILTIN_NANQ:
	case IX86_BUILTIN_NANSQ:
	  {
	    tree type = TREE_TYPE (TREE_TYPE (fndecl));
	    const char *str = c_getstr (*args);
	    int quiet = fn_code == IX86_BUILTIN_NANQ;
	    REAL_VALUE_TYPE real;

	    if (str && real_nan (&real, str, quiet, TYPE_MODE (type)))
	      return build_real (type, real);
	    return NULL_TREE;
	  }

	case IX86_BUILTIN_INFQ:
	case IX86_BUILTIN_HUGE_VALQ:
	  {
	    tree type = TREE_TYPE (TREE_TYPE (fndecl));
	    REAL_VALUE_TYPE inf;
	    real_inf (&inf);
	    return build_real (type, inf);
	  }

	case IX86_BUILTIN_TZCNT16:
	case IX86_BUILTIN_CTZS:
	case IX86_BUILTIN_TZCNT32:
	case IX86_BUILTIN_TZCNT64:
	  gcc_assert (n_args == 1);
	  if (TREE_CODE (args[0]) == INTEGER_CST)
	    {
	      tree type = TREE_TYPE (TREE_TYPE (fndecl));
	      tree arg = args[0];
	      if (fn_code == IX86_BUILTIN_TZCNT16
		  || fn_code == IX86_BUILTIN_CTZS)
		arg = fold_convert (short_unsigned_type_node, arg);
	      if (integer_zerop (arg))
		return build_int_cst (type, TYPE_PRECISION (TREE_TYPE (arg)));
	      else
		return fold_const_call (CFN_CTZ, type, arg);
	    }
	  break;

	case IX86_BUILTIN_LZCNT16:
	case IX86_BUILTIN_CLZS:
	case IX86_BUILTIN_LZCNT32:
	case IX86_BUILTIN_LZCNT64:
	  gcc_assert (n_args == 1);
	  if (TREE_CODE (args[0]) == INTEGER_CST)
	    {
	      tree type = TREE_TYPE (TREE_TYPE (fndecl));
	      tree arg = args[0];
	      if (fn_code == IX86_BUILTIN_LZCNT16
		  || fn_code == IX86_BUILTIN_CLZS)
		arg = fold_convert (short_unsigned_type_node, arg);
	      if (integer_zerop (arg))
		return build_int_cst (type, TYPE_PRECISION (TREE_TYPE (arg)));
	      else
		return fold_const_call (CFN_CLZ, type, arg);
	    }
	  break;

	case IX86_BUILTIN_BEXTR32:
	case IX86_BUILTIN_BEXTR64:
	case IX86_BUILTIN_BEXTRI32:
	case IX86_BUILTIN_BEXTRI64:
	  gcc_assert (n_args == 2);
	  if (tree_fits_uhwi_p (args[1]))
	    {
	      unsigned HOST_WIDE_INT res = 0;
	      unsigned int prec = TYPE_PRECISION (TREE_TYPE (args[0]));
	      unsigned int start = tree_to_uhwi (args[1]);
	      unsigned int len = (start & 0xff00) >> 8;
	      start &= 0xff;
	      if (start >= prec || len == 0)
		res = 0;
	      else if (!tree_fits_uhwi_p (args[0]))
		break;
	      else
		res = tree_to_uhwi (args[0]) >> start;
	      if (len > prec)
		len = prec;
	      if (len < HOST_BITS_PER_WIDE_INT)
		res &= (HOST_WIDE_INT_1U << len) - 1;
	      return build_int_cstu (TREE_TYPE (TREE_TYPE (fndecl)), res);
	    }
	  break;

	case IX86_BUILTIN_BZHI32:
	case IX86_BUILTIN_BZHI64:
	  gcc_assert (n_args == 2);
	  if (tree_fits_uhwi_p (args[1]))
	    {
	      unsigned int idx = tree_to_uhwi (args[1]) & 0xff;
	      if (idx >= TYPE_PRECISION (TREE_TYPE (args[0])))
		return args[0];
	      if (idx == 0)
		return build_int_cst (TREE_TYPE (TREE_TYPE (fndecl)), 0);
	      if (!tree_fits_uhwi_p (args[0]))
		break;
	      unsigned HOST_WIDE_INT res = tree_to_uhwi (args[0]);
	      res &= ~(HOST_WIDE_INT_M1U << idx);
	      return build_int_cstu (TREE_TYPE (TREE_TYPE (fndecl)), res);
	    }
	  break;

	case IX86_BUILTIN_PDEP32:
	case IX86_BUILTIN_PDEP64:
	  gcc_assert (n_args == 2);
	  if (tree_fits_uhwi_p (args[0]) && tree_fits_uhwi_p (args[1]))
	    {
	      unsigned HOST_WIDE_INT src = tree_to_uhwi (args[0]);
	      unsigned HOST_WIDE_INT mask = tree_to_uhwi (args[1]);
	      unsigned HOST_WIDE_INT res = 0;
	      unsigned HOST_WIDE_INT m, k = 1;
	      for (m = 1; m; m <<= 1)
		if ((mask & m) != 0)
		  {
		    if ((src & k) != 0)
		      res |= m;
		    k <<= 1;
		  }
	      return build_int_cstu (TREE_TYPE (TREE_TYPE (fndecl)), res);
	    }
	  break;

	case IX86_BUILTIN_PEXT32:
	case IX86_BUILTIN_PEXT64:
	  gcc_assert (n_args == 2);
	  if (tree_fits_uhwi_p (args[0]) && tree_fits_uhwi_p (args[1]))
	    {
	      unsigned HOST_WIDE_INT src = tree_to_uhwi (args[0]);
	      unsigned HOST_WIDE_INT mask = tree_to_uhwi (args[1]);
	      unsigned HOST_WIDE_INT res = 0;
	      unsigned HOST_WIDE_INT m, k = 1;
	      for (m = 1; m; m <<= 1)
		if ((mask & m) != 0)
		  {
		    if ((src & m) != 0)
		      res |= k;
		    k <<= 1;
		  }
	      return build_int_cstu (TREE_TYPE (TREE_TYPE (fndecl)), res);
	    }
	  break;

	case IX86_BUILTIN_MOVMSKPS:
	case IX86_BUILTIN_PMOVMSKB:
	case IX86_BUILTIN_MOVMSKPD:
	case IX86_BUILTIN_PMOVMSKB128:
	case IX86_BUILTIN_MOVMSKPD256:
	case IX86_BUILTIN_MOVMSKPS256:
	case IX86_BUILTIN_PMOVMSKB256:
	  gcc_assert (n_args == 1);
	  if (TREE_CODE (args[0]) == VECTOR_CST)
	    {
	      HOST_WIDE_INT res = 0;
	      for (unsigned i = 0; i < VECTOR_CST_NELTS (args[0]); ++i)
		{
		  tree e = VECTOR_CST_ELT (args[0], i);
		  if (TREE_CODE (e) == INTEGER_CST && !TREE_OVERFLOW (e))
		    {
		      if (wi::neg_p (wi::to_wide (e)))
			res |= HOST_WIDE_INT_1 << i;
		    }
		  else if (TREE_CODE (e) == REAL_CST && !TREE_OVERFLOW (e))
		    {
		      if (TREE_REAL_CST (e).sign)
			res |= HOST_WIDE_INT_1 << i;
		    }
		  else
		    return NULL_TREE;
		}
	      return build_int_cst (TREE_TYPE (TREE_TYPE (fndecl)), res);
	    }
	  break;

	case IX86_BUILTIN_PSLLD:
	case IX86_BUILTIN_PSLLD128:
	case IX86_BUILTIN_PSLLD128_MASK:
	case IX86_BUILTIN_PSLLD256:
	case IX86_BUILTIN_PSLLD256_MASK:
	case IX86_BUILTIN_PSLLD512:
	case IX86_BUILTIN_PSLLDI:
	case IX86_BUILTIN_PSLLDI128:
	case IX86_BUILTIN_PSLLDI128_MASK:
	case IX86_BUILTIN_PSLLDI256:
	case IX86_BUILTIN_PSLLDI256_MASK:
	case IX86_BUILTIN_PSLLDI512:
	case IX86_BUILTIN_PSLLQ:
	case IX86_BUILTIN_PSLLQ128:
	case IX86_BUILTIN_PSLLQ128_MASK:
	case IX86_BUILTIN_PSLLQ256:
	case IX86_BUILTIN_PSLLQ256_MASK:
	case IX86_BUILTIN_PSLLQ512:
	case IX86_BUILTIN_PSLLQI:
	case IX86_BUILTIN_PSLLQI128:
	case IX86_BUILTIN_PSLLQI128_MASK:
	case IX86_BUILTIN_PSLLQI256:
	case IX86_BUILTIN_PSLLQI256_MASK:
	case IX86_BUILTIN_PSLLQI512:
	case IX86_BUILTIN_PSLLW:
	case IX86_BUILTIN_PSLLW128:
	case IX86_BUILTIN_PSLLW128_MASK:
	case IX86_BUILTIN_PSLLW256:
	case IX86_BUILTIN_PSLLW256_MASK:
	case IX86_BUILTIN_PSLLW512_MASK:
	case IX86_BUILTIN_PSLLWI:
	case IX86_BUILTIN_PSLLWI128:
	case IX86_BUILTIN_PSLLWI128_MASK:
	case IX86_BUILTIN_PSLLWI256:
	case IX86_BUILTIN_PSLLWI256_MASK:
	case IX86_BUILTIN_PSLLWI512_MASK:
	  rcode = ASHIFT;
	  is_vshift = false;
	  goto do_shift;
	case IX86_BUILTIN_PSRAD:
	case IX86_BUILTIN_PSRAD128:
	case IX86_BUILTIN_PSRAD128_MASK:
	case IX86_BUILTIN_PSRAD256:
	case IX86_BUILTIN_PSRAD256_MASK:
	case IX86_BUILTIN_PSRAD512:
	case IX86_BUILTIN_PSRADI:
	case IX86_BUILTIN_PSRADI128:
	case IX86_BUILTIN_PSRADI128_MASK:
	case IX86_BUILTIN_PSRADI256:
	case IX86_BUILTIN_PSRADI256_MASK:
	case IX86_BUILTIN_PSRADI512:
	case IX86_BUILTIN_PSRAQ128_MASK:
	case IX86_BUILTIN_PSRAQ256_MASK:
	case IX86_BUILTIN_PSRAQ512:
	case IX86_BUILTIN_PSRAQI128_MASK:
	case IX86_BUILTIN_PSRAQI256_MASK:
	case IX86_BUILTIN_PSRAQI512:
	case IX86_BUILTIN_PSRAW:
	case IX86_BUILTIN_PSRAW128:
	case IX86_BUILTIN_PSRAW128_MASK:
	case IX86_BUILTIN_PSRAW256:
	case IX86_BUILTIN_PSRAW256_MASK:
	case IX86_BUILTIN_PSRAW512:
	case IX86_BUILTIN_PSRAWI:
	case IX86_BUILTIN_PSRAWI128:
	case IX86_BUILTIN_PSRAWI128_MASK:
	case IX86_BUILTIN_PSRAWI256:
	case IX86_BUILTIN_PSRAWI256_MASK:
	case IX86_BUILTIN_PSRAWI512:
	  rcode = ASHIFTRT;
	  is_vshift = false;
	  goto do_shift;
	case IX86_BUILTIN_PSRLD:
	case IX86_BUILTIN_PSRLD128:
	case IX86_BUILTIN_PSRLD128_MASK:
	case IX86_BUILTIN_PSRLD256:
	case IX86_BUILTIN_PSRLD256_MASK:
	case IX86_BUILTIN_PSRLD512:
	case IX86_BUILTIN_PSRLDI:
	case IX86_BUILTIN_PSRLDI128:
	case IX86_BUILTIN_PSRLDI128_MASK:
	case IX86_BUILTIN_PSRLDI256:
	case IX86_BUILTIN_PSRLDI256_MASK:
	case IX86_BUILTIN_PSRLDI512:
	case IX86_BUILTIN_PSRLQ:
	case IX86_BUILTIN_PSRLQ128:
	case IX86_BUILTIN_PSRLQ128_MASK:
	case IX86_BUILTIN_PSRLQ256:
	case IX86_BUILTIN_PSRLQ256_MASK:
	case IX86_BUILTIN_PSRLQ512:
	case IX86_BUILTIN_PSRLQI:
	case IX86_BUILTIN_PSRLQI128:
	case IX86_BUILTIN_PSRLQI128_MASK:
	case IX86_BUILTIN_PSRLQI256:
	case IX86_BUILTIN_PSRLQI256_MASK:
	case IX86_BUILTIN_PSRLQI512:
	case IX86_BUILTIN_PSRLW:
	case IX86_BUILTIN_PSRLW128:
	case IX86_BUILTIN_PSRLW128_MASK:
	case IX86_BUILTIN_PSRLW256:
	case IX86_BUILTIN_PSRLW256_MASK:
	case IX86_BUILTIN_PSRLW512:
	case IX86_BUILTIN_PSRLWI:
	case IX86_BUILTIN_PSRLWI128:
	case IX86_BUILTIN_PSRLWI128_MASK:
	case IX86_BUILTIN_PSRLWI256:
	case IX86_BUILTIN_PSRLWI256_MASK:
	case IX86_BUILTIN_PSRLWI512:
	  rcode = LSHIFTRT;
	  is_vshift = false;
	  goto do_shift;
	case IX86_BUILTIN_PSLLVV16HI:
	case IX86_BUILTIN_PSLLVV16SI:
	case IX86_BUILTIN_PSLLVV2DI:
	case IX86_BUILTIN_PSLLVV2DI_MASK:
	case IX86_BUILTIN_PSLLVV32HI:
	case IX86_BUILTIN_PSLLVV4DI:
	case IX86_BUILTIN_PSLLVV4DI_MASK:
	case IX86_BUILTIN_PSLLVV4SI:
	case IX86_BUILTIN_PSLLVV4SI_MASK:
	case IX86_BUILTIN_PSLLVV8DI:
	case IX86_BUILTIN_PSLLVV8HI:
	case IX86_BUILTIN_PSLLVV8SI:
	case IX86_BUILTIN_PSLLVV8SI_MASK:
	  rcode = ASHIFT;
	  is_vshift = true;
	  goto do_shift;
	case IX86_BUILTIN_PSRAVQ128:
	case IX86_BUILTIN_PSRAVQ256:
	case IX86_BUILTIN_PSRAVV16HI:
	case IX86_BUILTIN_PSRAVV16SI:
	case IX86_BUILTIN_PSRAVV32HI:
	case IX86_BUILTIN_PSRAVV4SI:
	case IX86_BUILTIN_PSRAVV4SI_MASK:
	case IX86_BUILTIN_PSRAVV8DI:
	case IX86_BUILTIN_PSRAVV8HI:
	case IX86_BUILTIN_PSRAVV8SI:
	case IX86_BUILTIN_PSRAVV8SI_MASK:
	  rcode = ASHIFTRT;
	  is_vshift = true;
	  goto do_shift;
	case IX86_BUILTIN_PSRLVV16HI:
	case IX86_BUILTIN_PSRLVV16SI:
	case IX86_BUILTIN_PSRLVV2DI:
	case IX86_BUILTIN_PSRLVV2DI_MASK:
	case IX86_BUILTIN_PSRLVV32HI:
	case IX86_BUILTIN_PSRLVV4DI:
	case IX86_BUILTIN_PSRLVV4DI_MASK:
	case IX86_BUILTIN_PSRLVV4SI:
	case IX86_BUILTIN_PSRLVV4SI_MASK:
	case IX86_BUILTIN_PSRLVV8DI:
	case IX86_BUILTIN_PSRLVV8HI:
	case IX86_BUILTIN_PSRLVV8SI:
	case IX86_BUILTIN_PSRLVV8SI_MASK:
	  rcode = LSHIFTRT;
	  is_vshift = true;
	  goto do_shift;

	do_shift:
	  gcc_assert (n_args >= 2);
	  if (TREE_CODE (args[0]) != VECTOR_CST)
	    break;
	  mask = HOST_WIDE_INT_M1U;
	  if (n_args > 2)
	    {
	      /* This is masked shift.  */
	      if (!tree_fits_uhwi_p (args[n_args - 1])
		  || TREE_SIDE_EFFECTS (args[n_args - 2]))
		break;
	      mask = tree_to_uhwi (args[n_args - 1]);
	      unsigned elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
	      mask |= HOST_WIDE_INT_M1U << elems;
	      if (mask != HOST_WIDE_INT_M1U
		  && TREE_CODE (args[n_args - 2]) != VECTOR_CST)
		break;
	      if (mask == (HOST_WIDE_INT_M1U << elems))
		return args[n_args - 2];
	    }
	  if (is_vshift && TREE_CODE (args[1]) != VECTOR_CST)
	    break;
	  if (tree tem = (is_vshift ? integer_one_node
			  : ix86_vector_shift_count (args[1])))
	    {
	      unsigned HOST_WIDE_INT count = tree_to_uhwi (tem);
	      unsigned HOST_WIDE_INT prec
		= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (args[0])));
	      if (count == 0 && mask == HOST_WIDE_INT_M1U)
		return args[0];
	      if (count >= prec)
		{
		  if (rcode == ASHIFTRT)
		    count = prec - 1;
		  else if (mask == HOST_WIDE_INT_M1U)
		    return build_zero_cst (TREE_TYPE (args[0]));
		}
	      tree countt = NULL_TREE;
	      if (!is_vshift)
		{
		  if (count >= prec)
		    countt = integer_zero_node;
		  else
		    countt = build_int_cst (integer_type_node, count);
		}
	      tree_vector_builder builder;
	      if (mask != HOST_WIDE_INT_M1U || is_vshift)
		builder.new_vector (TREE_TYPE (args[0]),
				    TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0])),
				    1);
	      else
		builder.new_unary_operation (TREE_TYPE (args[0]), args[0],
					     false);
	      unsigned int cnt = builder.encoded_nelts ();
	      for (unsigned int i = 0; i < cnt; ++i)
		{
		  tree elt = VECTOR_CST_ELT (args[0], i);
		  if (TREE_CODE (elt) != INTEGER_CST || TREE_OVERFLOW (elt))
		    return NULL_TREE;
		  tree type = TREE_TYPE (elt);
		  if (rcode == LSHIFTRT)
		    elt = fold_convert (unsigned_type_for (type), elt);
		  if (is_vshift)
		    {
		      countt = VECTOR_CST_ELT (args[1], i);
		      if (TREE_CODE (countt) != INTEGER_CST
			  || TREE_OVERFLOW (countt))
			return NULL_TREE;
		      if (wi::neg_p (wi::to_wide (countt))
			  || wi::to_widest (countt) >= prec)
			{
			  if (rcode == ASHIFTRT)
			    countt = build_int_cst (TREE_TYPE (countt),
						    prec - 1);
			  else
			    {
			      elt = build_zero_cst (TREE_TYPE (elt));
			      countt = build_zero_cst (TREE_TYPE (countt));
			    }
			}
		    }
		  else if (count >= prec)
		    elt = build_zero_cst (TREE_TYPE (elt));
		  elt = const_binop (rcode == ASHIFT
				     ? LSHIFT_EXPR : RSHIFT_EXPR,
				     TREE_TYPE (elt), elt, countt);
		  if (!elt || TREE_CODE (elt) != INTEGER_CST)
		    return NULL_TREE;
		  if (rcode == LSHIFTRT)
		    elt = fold_convert (type, elt);
		  if ((mask & (HOST_WIDE_INT_1U << i)) == 0)
		    {
		      elt = VECTOR_CST_ELT (args[n_args - 2], i);
		      if (TREE_CODE (elt) != INTEGER_CST
			  || TREE_OVERFLOW (elt))
			return NULL_TREE;
		    }
		  builder.quick_push (elt);
		}
	      return builder.build ();
	    }
	  break;

	default:
	  break;
	}
    }

#ifdef SUBTARGET_FOLD_BUILTIN
  return SUBTARGET_FOLD_BUILTIN (fndecl, n_args, args, ignore);
#endif

  return NULL_TREE;
}

/* Fold a MD builtin (use ix86_fold_builtin for folding into
   constant) in GIMPLE.  */

bool
ix86_gimple_fold_builtin (gimple_stmt_iterator *gsi)
{
  gimple *stmt = gsi_stmt (*gsi), *g;
  gimple_seq stmts = NULL;
  tree fndecl = gimple_call_fndecl (stmt);
  gcc_checking_assert (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_MD));
  int n_args = gimple_call_num_args (stmt);
  enum ix86_builtins fn_code
    = (enum ix86_builtins) DECL_MD_FUNCTION_CODE (fndecl);
  tree decl = NULL_TREE;
  tree arg0, arg1, arg2;
  enum rtx_code rcode;
  enum tree_code tcode;
  unsigned HOST_WIDE_INT count;
  bool is_vshift;
  unsigned HOST_WIDE_INT elems;
  location_t loc;

  /* Don't fold when there's isa mismatch.  */
  if (!ix86_check_builtin_isa_match (fn_code, NULL, NULL))
    return false;

  switch (fn_code)
    {
    case IX86_BUILTIN_TZCNT32:
      decl = builtin_decl_implicit (BUILT_IN_CTZ);
      goto fold_tzcnt_lzcnt;

    case IX86_BUILTIN_TZCNT64:
      decl = builtin_decl_implicit (BUILT_IN_CTZLL);
      goto fold_tzcnt_lzcnt;

    case IX86_BUILTIN_LZCNT32:
      decl = builtin_decl_implicit (BUILT_IN_CLZ);
      goto fold_tzcnt_lzcnt;

    case IX86_BUILTIN_LZCNT64:
      decl = builtin_decl_implicit (BUILT_IN_CLZLL);
      goto fold_tzcnt_lzcnt;

    fold_tzcnt_lzcnt:
      gcc_assert (n_args == 1);
      arg0 = gimple_call_arg (stmt, 0);
      if (TREE_CODE (arg0) == SSA_NAME && decl && gimple_call_lhs (stmt))
	{
	  int prec = TYPE_PRECISION (TREE_TYPE (arg0));
	  /* If arg0 is provably non-zero, optimize into generic
	     __builtin_c[tl]z{,ll} function the middle-end handles
	     better.  */
	  if (!expr_not_equal_to (arg0, wi::zero (prec)))
	    return false;

	  loc = gimple_location (stmt);
	  g = gimple_build_call (decl, 1, arg0);
	  gimple_set_location (g, loc);
	  tree lhs = make_ssa_name (integer_type_node);
	  gimple_call_set_lhs (g, lhs);
	  gsi_insert_before (gsi, g, GSI_SAME_STMT);
	  g = gimple_build_assign (gimple_call_lhs (stmt), NOP_EXPR, lhs);
	  gimple_set_location (g, loc);
	  gsi_replace (gsi, g, false);
	  return true;
	}
      break;

    case IX86_BUILTIN_BZHI32:
    case IX86_BUILTIN_BZHI64:
      gcc_assert (n_args == 2);
      arg1 = gimple_call_arg (stmt, 1);
      if (tree_fits_uhwi_p (arg1) && gimple_call_lhs (stmt))
	{
	  unsigned int idx = tree_to_uhwi (arg1) & 0xff;
	  arg0 = gimple_call_arg (stmt, 0);
	  if (idx < TYPE_PRECISION (TREE_TYPE (arg0)))
	    break;
	  loc = gimple_location (stmt);
	  g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
	  gimple_set_location (g, loc);
	  gsi_replace (gsi, g, false);
	  return true;
	}
      break;

    case IX86_BUILTIN_PDEP32:
    case IX86_BUILTIN_PDEP64:
    case IX86_BUILTIN_PEXT32:
    case IX86_BUILTIN_PEXT64:
      gcc_assert (n_args == 2);
      arg1 = gimple_call_arg (stmt, 1);
      if (integer_all_onesp (arg1) && gimple_call_lhs (stmt))
	{
	  loc = gimple_location (stmt);
	  arg0 = gimple_call_arg (stmt, 0);
	  g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
	  gimple_set_location (g, loc);
	  gsi_replace (gsi, g, false);
	  return true;
	}
      break;

    case IX86_BUILTIN_PBLENDVB256:
    case IX86_BUILTIN_BLENDVPS256:
    case IX86_BUILTIN_BLENDVPD256:
      /* pcmpeqb/d/q is under avx2, w/o avx2, it's veclower
	 to scalar operations and not combined back.  */
      if (!TARGET_AVX2)
	break;

      /* FALLTHRU.  */
    case IX86_BUILTIN_BLENDVPD:
      /* blendvpd is under sse4.1 but pcmpgtq is under sse4.2,
	 w/o sse4.2, it's veclowered to scalar operations and
	 not combined back.  */
      if (!TARGET_SSE4_2)
	break;
      /* FALLTHRU.  */
    case IX86_BUILTIN_PBLENDVB128:
    case IX86_BUILTIN_BLENDVPS:
      gcc_assert (n_args == 3);
      arg0 = gimple_call_arg (stmt, 0);
      arg1 = gimple_call_arg (stmt, 1);
      arg2 = gimple_call_arg (stmt, 2);
      if (gimple_call_lhs (stmt))
	{
	  loc = gimple_location (stmt);
	  tree type = TREE_TYPE (arg2);
	  if (VECTOR_FLOAT_TYPE_P (type))
	    {
	      tree itype = GET_MODE_INNER (TYPE_MODE (type)) == E_SFmode
		? intSI_type_node : intDI_type_node;
	      type = get_same_sized_vectype (itype, type);
	    }
	  else
	    type = signed_type_for (type);
	  arg2 = gimple_build (&stmts, VIEW_CONVERT_EXPR, type, arg2);
	  tree zero_vec = build_zero_cst (type);
	  tree cmp_type = truth_type_for (type);
	  tree cmp = gimple_build (&stmts, LT_EXPR, cmp_type, arg2, zero_vec);
	  gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
	  g = gimple_build_assign (gimple_call_lhs (stmt),
				   VEC_COND_EXPR, cmp,
				   arg1, arg0);
	  gimple_set_location (g, loc);
	  gsi_replace (gsi, g, false);
	}
      else
	gsi_replace (gsi, gimple_build_nop (), false);
      return true;


    case IX86_BUILTIN_PCMPEQB128:
    case IX86_BUILTIN_PCMPEQW128:
    case IX86_BUILTIN_PCMPEQD128:
    case IX86_BUILTIN_PCMPEQQ:
    case IX86_BUILTIN_PCMPEQB256:
    case IX86_BUILTIN_PCMPEQW256:
    case IX86_BUILTIN_PCMPEQD256:
    case IX86_BUILTIN_PCMPEQQ256:
      tcode = EQ_EXPR;
      goto do_cmp;

    case IX86_BUILTIN_PCMPGTB128:
    case IX86_BUILTIN_PCMPGTW128:
    case IX86_BUILTIN_PCMPGTD128:
    case IX86_BUILTIN_PCMPGTQ:
    case IX86_BUILTIN_PCMPGTB256:
    case IX86_BUILTIN_PCMPGTW256:
    case IX86_BUILTIN_PCMPGTD256:
    case IX86_BUILTIN_PCMPGTQ256:
      tcode = GT_EXPR;

    do_cmp:
      gcc_assert (n_args == 2);
      arg0 = gimple_call_arg (stmt, 0);
      arg1 = gimple_call_arg (stmt, 1);
      if (gimple_call_lhs (stmt))
	{
	  loc = gimple_location (stmt);
	  tree type = TREE_TYPE (arg0);
	  tree zero_vec = build_zero_cst (type);
	  tree minus_one_vec = build_minus_one_cst (type);
	  tree cmp_type = truth_type_for (type);
	  tree cmp = gimple_build (&stmts, tcode, cmp_type, arg0, arg1);
	  gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
	  g = gimple_build_assign (gimple_call_lhs (stmt),
				   VEC_COND_EXPR, cmp,
				   minus_one_vec, zero_vec);
	  gimple_set_location (g, loc);
	  gsi_replace (gsi, g, false);
	}
      else
	gsi_replace (gsi, gimple_build_nop (), false);
      return true;

    case IX86_BUILTIN_PSLLD:
    case IX86_BUILTIN_PSLLD128:
    case IX86_BUILTIN_PSLLD128_MASK:
    case IX86_BUILTIN_PSLLD256:
    case IX86_BUILTIN_PSLLD256_MASK:
    case IX86_BUILTIN_PSLLD512:
    case IX86_BUILTIN_PSLLDI:
    case IX86_BUILTIN_PSLLDI128:
    case IX86_BUILTIN_PSLLDI128_MASK:
    case IX86_BUILTIN_PSLLDI256:
    case IX86_BUILTIN_PSLLDI256_MASK:
    case IX86_BUILTIN_PSLLDI512:
    case IX86_BUILTIN_PSLLQ:
    case IX86_BUILTIN_PSLLQ128:
    case IX86_BUILTIN_PSLLQ128_MASK:
    case IX86_BUILTIN_PSLLQ256:
    case IX86_BUILTIN_PSLLQ256_MASK:
    case IX86_BUILTIN_PSLLQ512:
    case IX86_BUILTIN_PSLLQI:
    case IX86_BUILTIN_PSLLQI128:
    case IX86_BUILTIN_PSLLQI128_MASK:
    case IX86_BUILTIN_PSLLQI256:
    case IX86_BUILTIN_PSLLQI256_MASK:
    case IX86_BUILTIN_PSLLQI512:
    case IX86_BUILTIN_PSLLW:
    case IX86_BUILTIN_PSLLW128:
    case IX86_BUILTIN_PSLLW128_MASK:
    case IX86_BUILTIN_PSLLW256:
    case IX86_BUILTIN_PSLLW256_MASK:
    case IX86_BUILTIN_PSLLW512_MASK:
    case IX86_BUILTIN_PSLLWI:
    case IX86_BUILTIN_PSLLWI128:
    case IX86_BUILTIN_PSLLWI128_MASK:
    case IX86_BUILTIN_PSLLWI256:
    case IX86_BUILTIN_PSLLWI256_MASK:
    case IX86_BUILTIN_PSLLWI512_MASK:
      rcode = ASHIFT;
      is_vshift = false;
      goto do_shift;
    case IX86_BUILTIN_PSRAD:
    case IX86_BUILTIN_PSRAD128:
    case IX86_BUILTIN_PSRAD128_MASK:
    case IX86_BUILTIN_PSRAD256:
    case IX86_BUILTIN_PSRAD256_MASK:
    case IX86_BUILTIN_PSRAD512:
    case IX86_BUILTIN_PSRADI:
    case IX86_BUILTIN_PSRADI128:
    case IX86_BUILTIN_PSRADI128_MASK:
    case IX86_BUILTIN_PSRADI256:
    case IX86_BUILTIN_PSRADI256_MASK:
    case IX86_BUILTIN_PSRADI512:
    case IX86_BUILTIN_PSRAQ128_MASK:
    case IX86_BUILTIN_PSRAQ256_MASK:
    case IX86_BUILTIN_PSRAQ512:
    case IX86_BUILTIN_PSRAQI128_MASK:
    case IX86_BUILTIN_PSRAQI256_MASK:
    case IX86_BUILTIN_PSRAQI512:
    case IX86_BUILTIN_PSRAW:
    case IX86_BUILTIN_PSRAW128:
    case IX86_BUILTIN_PSRAW128_MASK:
    case IX86_BUILTIN_PSRAW256:
    case IX86_BUILTIN_PSRAW256_MASK:
    case IX86_BUILTIN_PSRAW512:
    case IX86_BUILTIN_PSRAWI:
    case IX86_BUILTIN_PSRAWI128:
    case IX86_BUILTIN_PSRAWI128_MASK:
    case IX86_BUILTIN_PSRAWI256:
    case IX86_BUILTIN_PSRAWI256_MASK:
    case IX86_BUILTIN_PSRAWI512:
      rcode = ASHIFTRT;
      is_vshift = false;
      goto do_shift;
    case IX86_BUILTIN_PSRLD:
    case IX86_BUILTIN_PSRLD128:
    case IX86_BUILTIN_PSRLD128_MASK:
    case IX86_BUILTIN_PSRLD256:
    case IX86_BUILTIN_PSRLD256_MASK:
    case IX86_BUILTIN_PSRLD512:
    case IX86_BUILTIN_PSRLDI:
    case IX86_BUILTIN_PSRLDI128:
    case IX86_BUILTIN_PSRLDI128_MASK:
    case IX86_BUILTIN_PSRLDI256:
    case IX86_BUILTIN_PSRLDI256_MASK:
    case IX86_BUILTIN_PSRLDI512:
    case IX86_BUILTIN_PSRLQ:
    case IX86_BUILTIN_PSRLQ128:
    case IX86_BUILTIN_PSRLQ128_MASK:
    case IX86_BUILTIN_PSRLQ256:
    case IX86_BUILTIN_PSRLQ256_MASK:
    case IX86_BUILTIN_PSRLQ512:
    case IX86_BUILTIN_PSRLQI:
    case IX86_BUILTIN_PSRLQI128:
    case IX86_BUILTIN_PSRLQI128_MASK:
    case IX86_BUILTIN_PSRLQI256:
    case IX86_BUILTIN_PSRLQI256_MASK:
    case IX86_BUILTIN_PSRLQI512:
    case IX86_BUILTIN_PSRLW:
    case IX86_BUILTIN_PSRLW128:
    case IX86_BUILTIN_PSRLW128_MASK:
    case IX86_BUILTIN_PSRLW256:
    case IX86_BUILTIN_PSRLW256_MASK:
    case IX86_BUILTIN_PSRLW512:
    case IX86_BUILTIN_PSRLWI:
    case IX86_BUILTIN_PSRLWI128:
    case IX86_BUILTIN_PSRLWI128_MASK:
    case IX86_BUILTIN_PSRLWI256:
    case IX86_BUILTIN_PSRLWI256_MASK:
    case IX86_BUILTIN_PSRLWI512:
      rcode = LSHIFTRT;
      is_vshift = false;
      goto do_shift;
    case IX86_BUILTIN_PSLLVV16HI:
    case IX86_BUILTIN_PSLLVV16SI:
    case IX86_BUILTIN_PSLLVV2DI:
    case IX86_BUILTIN_PSLLVV2DI_MASK:
    case IX86_BUILTIN_PSLLVV32HI:
    case IX86_BUILTIN_PSLLVV4DI:
    case IX86_BUILTIN_PSLLVV4DI_MASK:
    case IX86_BUILTIN_PSLLVV4SI:
    case IX86_BUILTIN_PSLLVV4SI_MASK:
    case IX86_BUILTIN_PSLLVV8DI:
    case IX86_BUILTIN_PSLLVV8HI:
    case IX86_BUILTIN_PSLLVV8SI:
    case IX86_BUILTIN_PSLLVV8SI_MASK:
      rcode = ASHIFT;
      is_vshift = true;
      goto do_shift;
    case IX86_BUILTIN_PSRAVQ128:
    case IX86_BUILTIN_PSRAVQ256:
    case IX86_BUILTIN_PSRAVV16HI:
    case IX86_BUILTIN_PSRAVV16SI:
    case IX86_BUILTIN_PSRAVV32HI:
    case IX86_BUILTIN_PSRAVV4SI:
    case IX86_BUILTIN_PSRAVV4SI_MASK:
    case IX86_BUILTIN_PSRAVV8DI:
    case IX86_BUILTIN_PSRAVV8HI:
    case IX86_BUILTIN_PSRAVV8SI:
    case IX86_BUILTIN_PSRAVV8SI_MASK:
      rcode = ASHIFTRT;
      is_vshift = true;
      goto do_shift;
    case IX86_BUILTIN_PSRLVV16HI:
    case IX86_BUILTIN_PSRLVV16SI:
    case IX86_BUILTIN_PSRLVV2DI:
    case IX86_BUILTIN_PSRLVV2DI_MASK:
    case IX86_BUILTIN_PSRLVV32HI:
    case IX86_BUILTIN_PSRLVV4DI:
    case IX86_BUILTIN_PSRLVV4DI_MASK:
    case IX86_BUILTIN_PSRLVV4SI:
    case IX86_BUILTIN_PSRLVV4SI_MASK:
    case IX86_BUILTIN_PSRLVV8DI:
    case IX86_BUILTIN_PSRLVV8HI:
    case IX86_BUILTIN_PSRLVV8SI:
    case IX86_BUILTIN_PSRLVV8SI_MASK:
      rcode = LSHIFTRT;
      is_vshift = true;
      goto do_shift;

    do_shift:
      gcc_assert (n_args >= 2);
      if (!gimple_call_lhs (stmt))
	break;
      arg0 = gimple_call_arg (stmt, 0);
      arg1 = gimple_call_arg (stmt, 1);
      elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
      /* For masked shift, only optimize if the mask is all ones.  */
      if (n_args > 2
	  && !ix86_masked_all_ones (elems, gimple_call_arg (stmt, n_args - 1)))
	break;
      if (is_vshift)
	{
	  if (TREE_CODE (arg1) != VECTOR_CST)
	    break;
	  count = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (arg0)));
	  if (integer_zerop (arg1))
	    count = 0;
	  else if (rcode == ASHIFTRT)
	    break;
	  else
	    for (unsigned int i = 0; i < VECTOR_CST_NELTS (arg1); ++i)
	      {
		tree elt = VECTOR_CST_ELT (arg1, i);
		if (!wi::neg_p (wi::to_wide (elt))
		    && wi::to_widest (elt) < count)
		  return false;
	      }
	}
      else
	{
	  arg1 = ix86_vector_shift_count (arg1);
	  if (!arg1)
	    break;
	  count = tree_to_uhwi (arg1);
	}
      if (count == 0)
	{
	  /* Just return the first argument for shift by 0.  */
	  loc = gimple_location (stmt);
	  g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
	  gimple_set_location (g, loc);
	  gsi_replace (gsi, g, false);
	  return true;
	}
      if (rcode != ASHIFTRT
	  && count >= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (arg0))))
	{
	  /* For shift counts equal or greater than precision, except for
	     arithmetic right shift the result is zero.  */
	  loc = gimple_location (stmt);
	  g = gimple_build_assign (gimple_call_lhs (stmt),
				   build_zero_cst (TREE_TYPE (arg0)));
	  gimple_set_location (g, loc);
	  gsi_replace (gsi, g, false);
	  return true;
	}
      break;

    case IX86_BUILTIN_SHUFPD512:
    case IX86_BUILTIN_SHUFPS512:
    case IX86_BUILTIN_SHUFPD:
    case IX86_BUILTIN_SHUFPD256:
    case IX86_BUILTIN_SHUFPS:
    case IX86_BUILTIN_SHUFPS256:
      arg0 = gimple_call_arg (stmt, 0);
      elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
      /* This is masked shuffle.  Only optimize if the mask is all ones.  */
      if (n_args > 3
	  && !ix86_masked_all_ones (elems,
				    gimple_call_arg (stmt, n_args - 1)))
	break;
      arg2 = gimple_call_arg (stmt, 2);
      if (TREE_CODE (arg2) == INTEGER_CST && gimple_call_lhs (stmt))
	{
	  unsigned HOST_WIDE_INT shuffle_mask = TREE_INT_CST_LOW (arg2);
	  /* Check valid imm, refer to gcc.target/i386/testimm-10.c.  */
	  if (shuffle_mask > 255)
	    return false;

	  machine_mode imode = GET_MODE_INNER (TYPE_MODE (TREE_TYPE (arg0)));
	  loc = gimple_location (stmt);
	  tree itype = (imode == E_DFmode
			? long_long_integer_type_node : integer_type_node);
	  tree vtype = build_vector_type (itype, elems);
	  tree_vector_builder elts (vtype, elems, 1);


	  /* Transform integer shuffle_mask to vector perm_mask which
	     is used by vec_perm_expr, refer to shuflp[sd]256/512 in sse.md.  */
	  for (unsigned i = 0; i != elems; i++)
	    {
	      unsigned sel_idx;
	      /* Imm[1:0](if VL > 128, then use Imm[3:2],Imm[5:4],Imm[7:6])
		 provide 2 select constrols for each element of the
		 destination.  */
	      if (imode == E_DFmode)
		sel_idx = (i & 1) * elems + (i & ~1)
			  + ((shuffle_mask >> i) & 1);
	      else
		{
		  /* Imm[7:0](if VL > 128, also use Imm[7:0]) provide 4 select
		     controls for each element of the destination.  */
		  unsigned j = i % 4;
		  sel_idx = ((i >> 1) & 1) * elems + (i & ~3)
			    + ((shuffle_mask >> 2 * j) & 3);
		}
	      elts.quick_push (build_int_cst (itype, sel_idx));
	    }

	  tree perm_mask = elts.build ();
	  arg1 = gimple_call_arg (stmt, 1);
	  g = gimple_build_assign (gimple_call_lhs (stmt),
				   VEC_PERM_EXPR,
				   arg0, arg1, perm_mask);
	  gimple_set_location (g, loc);
	  gsi_replace (gsi, g, false);
	  return true;
	}
      // Do not error yet, the constant could be propagated later?
      break;

    case IX86_BUILTIN_PABSB:
    case IX86_BUILTIN_PABSW:
    case IX86_BUILTIN_PABSD:
      /* 64-bit vector abs<mode>2 is only supported under TARGET_MMX_WITH_SSE.  */
      if (!TARGET_MMX_WITH_SSE)
	break;
      /* FALLTHRU.  */
    case IX86_BUILTIN_PABSB128:
    case IX86_BUILTIN_PABSB256:
    case IX86_BUILTIN_PABSB512:
    case IX86_BUILTIN_PABSW128:
    case IX86_BUILTIN_PABSW256:
    case IX86_BUILTIN_PABSW512:
    case IX86_BUILTIN_PABSD128:
    case IX86_BUILTIN_PABSD256:
    case IX86_BUILTIN_PABSD512:
    case IX86_BUILTIN_PABSQ128:
    case IX86_BUILTIN_PABSQ256:
    case IX86_BUILTIN_PABSQ512:
    case IX86_BUILTIN_PABSB128_MASK:
    case IX86_BUILTIN_PABSB256_MASK:
    case IX86_BUILTIN_PABSW128_MASK:
    case IX86_BUILTIN_PABSW256_MASK:
    case IX86_BUILTIN_PABSD128_MASK:
    case IX86_BUILTIN_PABSD256_MASK:
      gcc_assert (n_args >= 1);
      if (!gimple_call_lhs (stmt))
	break;
      arg0 = gimple_call_arg (stmt, 0);
      elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
      /* For masked ABS, only optimize if the mask is all ones.  */
      if (n_args > 1
	  && !ix86_masked_all_ones (elems, gimple_call_arg (stmt, n_args - 1)))
	break;
      {
	tree utype, ures, vce;
	utype = unsigned_type_for (TREE_TYPE (arg0));
	/* PABSB/W/D/Q store the unsigned result in dst, use ABSU_EXPR
	   instead of ABS_EXPR to hanlde overflow case(TYPE_MIN).  */
	ures = gimple_build (&stmts, ABSU_EXPR, utype, arg0);
	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
	loc = gimple_location (stmt);
	vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg0), ures);
	g = gimple_build_assign (gimple_call_lhs (stmt),
				 VIEW_CONVERT_EXPR, vce);
	gsi_replace (gsi, g, false);
      }
      return true;

    default:
      break;
    }

  return false;
}

/* Handler for an SVML-style interface to
   a library with vectorized intrinsics.  */

tree
ix86_veclibabi_svml (combined_fn fn, tree type_out, tree type_in)
{
  char name[20];
  tree fntype, new_fndecl, args;
  unsigned arity;
  const char *bname;
  machine_mode el_mode, in_mode;
  int n, in_n;

  /* The SVML is suitable for unsafe math only.  */
  if (!flag_unsafe_math_optimizations)
    return NULL_TREE;

  el_mode = TYPE_MODE (TREE_TYPE (type_out));
  n = TYPE_VECTOR_SUBPARTS (type_out);
  in_mode = TYPE_MODE (TREE_TYPE (type_in));
  in_n = TYPE_VECTOR_SUBPARTS (type_in);
  if (el_mode != in_mode
      || n != in_n)
    return NULL_TREE;

  switch (fn)
    {
    CASE_CFN_EXP:
    CASE_CFN_LOG:
    CASE_CFN_LOG10:
    CASE_CFN_POW:
    CASE_CFN_TANH:
    CASE_CFN_TAN:
    CASE_CFN_ATAN:
    CASE_CFN_ATAN2:
    CASE_CFN_ATANH:
    CASE_CFN_CBRT:
    CASE_CFN_SINH:
    CASE_CFN_SIN:
    CASE_CFN_ASINH:
    CASE_CFN_ASIN:
    CASE_CFN_COSH:
    CASE_CFN_COS:
    CASE_CFN_ACOSH:
    CASE_CFN_ACOS:
      if ((el_mode != DFmode || n != 2)
	  && (el_mode != SFmode || n != 4))
	return NULL_TREE;
      break;

    default:
      return NULL_TREE;
    }

  tree fndecl = mathfn_built_in (el_mode == DFmode
				 ? double_type_node : float_type_node, fn);
  bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));

  if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LOGF)
    strcpy (name, "vmlsLn4");
  else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LOG)
    strcpy (name, "vmldLn2");
  else if (n == 4)
    {
      sprintf (name, "vmls%s", bname+10);
      name[strlen (name)-1] = '4';
    }
  else
    sprintf (name, "vmld%s2", bname+10);

  /* Convert to uppercase. */
  name[4] &= ~0x20;

  arity = 0;
  for (args = DECL_ARGUMENTS (fndecl); args; args = TREE_CHAIN (args))
    arity++;

  if (arity == 1)
    fntype = build_function_type_list (type_out, type_in, NULL);
  else
    fntype = build_function_type_list (type_out, type_in, type_in, NULL);

  /* Build a function declaration for the vectorized function.  */
  new_fndecl = build_decl (BUILTINS_LOCATION,
			   FUNCTION_DECL, get_identifier (name), fntype);
  TREE_PUBLIC (new_fndecl) = 1;
  DECL_EXTERNAL (new_fndecl) = 1;
  DECL_IS_NOVOPS (new_fndecl) = 1;
  TREE_READONLY (new_fndecl) = 1;

  return new_fndecl;
}

/* Handler for an ACML-style interface to
   a library with vectorized intrinsics.  */

tree
ix86_veclibabi_acml (combined_fn fn, tree type_out, tree type_in)
{
  char name[20] = "__vr.._";
  tree fntype, new_fndecl, args;
  unsigned arity;
  const char *bname;
  machine_mode el_mode, in_mode;
  int n, in_n;

  /* The ACML is 64bits only and suitable for unsafe math only as
     it does not correctly support parts of IEEE with the required
     precision such as denormals.  */
  if (!TARGET_64BIT
      || !flag_unsafe_math_optimizations)
    return NULL_TREE;

  el_mode = TYPE_MODE (TREE_TYPE (type_out));
  n = TYPE_VECTOR_SUBPARTS (type_out);
  in_mode = TYPE_MODE (TREE_TYPE (type_in));
  in_n = TYPE_VECTOR_SUBPARTS (type_in);
  if (el_mode != in_mode
      || n != in_n)
    return NULL_TREE;

  switch (fn)
    {
    CASE_CFN_SIN:
    CASE_CFN_COS:
    CASE_CFN_EXP:
    CASE_CFN_LOG:
    CASE_CFN_LOG2:
    CASE_CFN_LOG10:
      if (el_mode == DFmode && n == 2)
	{
	  name[4] = 'd';
	  name[5] = '2';
	}
      else if (el_mode == SFmode && n == 4)
	{
	  name[4] = 's';
	  name[5] = '4';
	}
      else
	return NULL_TREE;
      break;

    default:
      return NULL_TREE;
    }

  tree fndecl = mathfn_built_in (el_mode == DFmode
				 ? double_type_node : float_type_node, fn);
  bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
  sprintf (name + 7, "%s", bname+10);

  arity = 0;
  for (args = DECL_ARGUMENTS (fndecl); args; args = TREE_CHAIN (args))
    arity++;

  if (arity == 1)
    fntype = build_function_type_list (type_out, type_in, NULL);
  else
    fntype = build_function_type_list (type_out, type_in, type_in, NULL);

  /* Build a function declaration for the vectorized function.  */
  new_fndecl = build_decl (BUILTINS_LOCATION,
			   FUNCTION_DECL, get_identifier (name), fntype);
  TREE_PUBLIC (new_fndecl) = 1;
  DECL_EXTERNAL (new_fndecl) = 1;
  DECL_IS_NOVOPS (new_fndecl) = 1;
  TREE_READONLY (new_fndecl) = 1;

  return new_fndecl;
}

/* Returns a decl of a function that implements scatter store with
   register type VECTYPE and index type INDEX_TYPE and SCALE.
   Return NULL_TREE if it is not available.  */

static tree
ix86_vectorize_builtin_scatter (const_tree vectype,
				const_tree index_type, int scale)
{
  bool si;
  enum ix86_builtins code;

  if (!TARGET_AVX512F)
    return NULL_TREE;

  if (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 2u)
      ? !TARGET_USE_SCATTER_2PARTS
      : (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 4u)
	 ? !TARGET_USE_SCATTER_4PARTS
	 : !TARGET_USE_SCATTER_8PARTS))
    return NULL_TREE;

  if ((TREE_CODE (index_type) != INTEGER_TYPE
       && !POINTER_TYPE_P (index_type))
      || (TYPE_MODE (index_type) != SImode
	  && TYPE_MODE (index_type) != DImode))
    return NULL_TREE;

  if (TYPE_PRECISION (index_type) > POINTER_SIZE)
    return NULL_TREE;

  /* v*scatter* insn sign extends index to pointer mode.  */
  if (TYPE_PRECISION (index_type) < POINTER_SIZE
      && TYPE_UNSIGNED (index_type))
    return NULL_TREE;

  /* Scale can be 1, 2, 4 or 8.  */
  if (scale <= 0
      || scale > 8
      || (scale & (scale - 1)) != 0)
    return NULL_TREE;

  si = TYPE_MODE (index_type) == SImode;
  switch (TYPE_MODE (vectype))
    {
    case E_V8DFmode:
      code = si ? IX86_BUILTIN_SCATTERALTSIV8DF : IX86_BUILTIN_SCATTERDIV8DF;
      break;
    case E_V8DImode:
      code = si ? IX86_BUILTIN_SCATTERALTSIV8DI : IX86_BUILTIN_SCATTERDIV8DI;
      break;
    case E_V16SFmode:
      code = si ? IX86_BUILTIN_SCATTERSIV16SF : IX86_BUILTIN_SCATTERALTDIV16SF;
      break;
    case E_V16SImode:
      code = si ? IX86_BUILTIN_SCATTERSIV16SI : IX86_BUILTIN_SCATTERALTDIV16SI;
      break;
    case E_V4DFmode:
      if (TARGET_AVX512VL)
	code = si ? IX86_BUILTIN_SCATTERALTSIV4DF : IX86_BUILTIN_SCATTERDIV4DF;
      else
	return NULL_TREE;
      break;
    case E_V4DImode:
      if (TARGET_AVX512VL)
	code = si ? IX86_BUILTIN_SCATTERALTSIV4DI : IX86_BUILTIN_SCATTERDIV4DI;
      else
	return NULL_TREE;
      break;
    case E_V8SFmode:
      if (TARGET_AVX512VL)
	code = si ? IX86_BUILTIN_SCATTERSIV8SF : IX86_BUILTIN_SCATTERALTDIV8SF;
      else
	return NULL_TREE;
      break;
    case E_V8SImode:
      if (TARGET_AVX512VL)
	code = si ? IX86_BUILTIN_SCATTERSIV8SI : IX86_BUILTIN_SCATTERALTDIV8SI;
      else
	return NULL_TREE;
      break;
    case E_V2DFmode:
      if (TARGET_AVX512VL)
	code = si ? IX86_BUILTIN_SCATTERALTSIV2DF : IX86_BUILTIN_SCATTERDIV2DF;
      else
	return NULL_TREE;
      break;
    case E_V2DImode:
      if (TARGET_AVX512VL)
	code = si ? IX86_BUILTIN_SCATTERALTSIV2DI : IX86_BUILTIN_SCATTERDIV2DI;
      else
	return NULL_TREE;
      break;
    case E_V4SFmode:
      if (TARGET_AVX512VL)
	code = si ? IX86_BUILTIN_SCATTERSIV4SF : IX86_BUILTIN_SCATTERALTDIV4SF;
      else
	return NULL_TREE;
      break;
    case E_V4SImode:
      if (TARGET_AVX512VL)
	code = si ? IX86_BUILTIN_SCATTERSIV4SI : IX86_BUILTIN_SCATTERALTDIV4SI;
      else
	return NULL_TREE;
      break;
    default:
      return NULL_TREE;
    }

  return get_ix86_builtin (code);
}

/* Return true if it is safe to use the rsqrt optabs to optimize
   1.0/sqrt.  */

static bool
use_rsqrt_p (machine_mode mode)
{
  return ((mode == HFmode
	   || (TARGET_SSE && TARGET_SSE_MATH))
	  && flag_finite_math_only
	  && !flag_trapping_math
	  && flag_unsafe_math_optimizations);
}

/* Helper for avx_vpermilps256_operand et al.  This is also used by
   the expansion functions to turn the parallel back into a mask.
   The return value is 0 for no match and the imm8+1 for a match.  */

int
avx_vpermilp_parallel (rtx par, machine_mode mode)
{
  unsigned i, nelt = GET_MODE_NUNITS (mode);
  unsigned mask = 0;
  unsigned char ipar[16] = {};  /* Silence -Wuninitialized warning.  */

  if (XVECLEN (par, 0) != (int) nelt)
    return 0;

  /* Validate that all of the elements are constants, and not totally
     out of range.  Copy the data into an integral array to make the
     subsequent checks easier.  */
  for (i = 0; i < nelt; ++i)
    {
      rtx er = XVECEXP (par, 0, i);
      unsigned HOST_WIDE_INT ei;

      if (!CONST_INT_P (er))
	return 0;
      ei = INTVAL (er);
      if (ei >= nelt)
	return 0;
      ipar[i] = ei;
    }

  switch (mode)
    {
    case E_V8DFmode:
      /* In the 512-bit DFmode case, we can only move elements within
         a 128-bit lane.  First fill the second part of the mask,
	 then fallthru.  */
      for (i = 4; i < 6; ++i)
	{
	  if (ipar[i] < 4 || ipar[i] >= 6)
	    return 0;
	  mask |= (ipar[i] - 4) << i;
	}
      for (i = 6; i < 8; ++i)
	{
	  if (ipar[i] < 6)
	    return 0;
	  mask |= (ipar[i] - 6) << i;
	}
      /* FALLTHRU */

    case E_V4DFmode:
      /* In the 256-bit DFmode case, we can only move elements within
         a 128-bit lane.  */
      for (i = 0; i < 2; ++i)
	{
	  if (ipar[i] >= 2)
	    return 0;
	  mask |= ipar[i] << i;
	}
      for (i = 2; i < 4; ++i)
	{
	  if (ipar[i] < 2)
	    return 0;
	  mask |= (ipar[i] - 2) << i;
	}
      break;

    case E_V16SFmode:
      /* In 512 bit SFmode case, permutation in the upper 256 bits
	 must mirror the permutation in the lower 256-bits.  */
      for (i = 0; i < 8; ++i)
	if (ipar[i] + 8 != ipar[i + 8])
	  return 0;
      /* FALLTHRU */

    case E_V8SFmode:
      /* In 256 bit SFmode case, we have full freedom of
         movement within the low 128-bit lane, but the high 128-bit
         lane must mirror the exact same pattern.  */
      for (i = 0; i < 4; ++i)
	if (ipar[i] + 4 != ipar[i + 4])
	  return 0;
      nelt = 4;
      /* FALLTHRU */

    case E_V2DFmode:
    case E_V4SFmode:
      /* In the 128-bit case, we've full freedom in the placement of
	 the elements from the source operand.  */
      for (i = 0; i < nelt; ++i)
	mask |= ipar[i] << (i * (nelt / 2));
      break;

    default:
      gcc_unreachable ();
    }

  /* Make sure success has a non-zero value by adding one.  */
  return mask + 1;
}

/* Helper for avx_vperm2f128_v4df_operand et al.  This is also used by
   the expansion functions to turn the parallel back into a mask.
   The return value is 0 for no match and the imm8+1 for a match.  */

int
avx_vperm2f128_parallel (rtx par, machine_mode mode)
{
  unsigned i, nelt = GET_MODE_NUNITS (mode), nelt2 = nelt / 2;
  unsigned mask = 0;
  unsigned char ipar[8] = {};  /* Silence -Wuninitialized warning.  */

  if (XVECLEN (par, 0) != (int) nelt)
    return 0;

  /* Validate that all of the elements are constants, and not totally
     out of range.  Copy the data into an integral array to make the
     subsequent checks easier.  */
  for (i = 0; i < nelt; ++i)
    {
      rtx er = XVECEXP (par, 0, i);
      unsigned HOST_WIDE_INT ei;

      if (!CONST_INT_P (er))
	return 0;
      ei = INTVAL (er);
      if (ei >= 2 * nelt)
	return 0;
      ipar[i] = ei;
    }

  /* Validate that the halves of the permute are halves.  */
  for (i = 0; i < nelt2 - 1; ++i)
    if (ipar[i] + 1 != ipar[i + 1])
      return 0;
  for (i = nelt2; i < nelt - 1; ++i)
    if (ipar[i] + 1 != ipar[i + 1])
      return 0;

  /* Reconstruct the mask.  */
  for (i = 0; i < 2; ++i)
    {
      unsigned e = ipar[i * nelt2];
      if (e % nelt2)
	return 0;
      e /= nelt2;
      mask |= e << (i * 4);
    }

  /* Make sure success has a non-zero value by adding one.  */
  return mask + 1;
}

/* Return a mask of VPTERNLOG operands that do not affect output.  */

int
vpternlog_redundant_operand_mask (rtx pternlog_imm)
{
  int mask = 0;
  int imm8 = INTVAL (pternlog_imm);

  if (((imm8 >> 4) & 0x0F) == (imm8 & 0x0F))
    mask |= 1;
  if (((imm8 >> 2) & 0x33) == (imm8 & 0x33))
    mask |= 2;
  if (((imm8 >> 1) & 0x55) == (imm8 & 0x55))
    mask |= 4;

  return mask;
}

/* Eliminate false dependencies on operands that do not affect output
   by substituting other operands of a VPTERNLOG.  */

void
substitute_vpternlog_operands (rtx *operands)
{
  int mask = vpternlog_redundant_operand_mask (operands[4]);

  if (mask & 1) /* The first operand is redundant.  */
    operands[1] = operands[2];

  if (mask & 2) /* The second operand is redundant.  */
    operands[2] = operands[1];

  if (mask & 4) /* The third operand is redundant.  */
    operands[3] = operands[1];
  else if (REG_P (operands[3]))
    {
      if (mask & 1)
	operands[1] = operands[3];
      if (mask & 2)
	operands[2] = operands[3];
    }
}

/* Return a register priority for hard reg REGNO.  */
static int
ix86_register_priority (int hard_regno)
{
  /* ebp and r13 as the base always wants a displacement, r12 as the
     base always wants an index.  So discourage their usage in an
     address.  */
  if (hard_regno == R12_REG || hard_regno == R13_REG)
    return 0;
  if (hard_regno == BP_REG)
    return 1;
  /* New x86-64 int registers result in bigger code size.  Discourage them.  */
  if (REX_INT_REGNO_P (hard_regno))
    return 2;
  if (REX2_INT_REGNO_P (hard_regno))
    return 2;
  /* New x86-64 SSE registers result in bigger code size.  Discourage them.  */
  if (REX_SSE_REGNO_P (hard_regno))
    return 2;
  if (EXT_REX_SSE_REGNO_P (hard_regno))
    return 1;
  /* Usage of AX register results in smaller code.  Prefer it.  */
  if (hard_regno == AX_REG)
    return 4;
  return 3;
}

/* Implement TARGET_PREFERRED_RELOAD_CLASS.

   Put float CONST_DOUBLE in the constant pool instead of fp regs.
   QImode must go into class Q_REGS.
   Narrow ALL_REGS to GENERAL_REGS.  This supports allowing movsf and
   movdf to do mem-to-mem moves through integer regs.  */

static reg_class_t
ix86_preferred_reload_class (rtx x, reg_class_t regclass)
{
  machine_mode mode = GET_MODE (x);

  /* We're only allowed to return a subclass of CLASS.  Many of the
     following checks fail for NO_REGS, so eliminate that early.  */
  if (regclass == NO_REGS)
    return NO_REGS;

  /* All classes can load zeros.  */
  if (x == CONST0_RTX (mode))
    return regclass;

  /* Force constants into memory if we are loading a (nonzero) constant into
     an MMX, SSE or MASK register.  This is because there are no MMX/SSE/MASK
     instructions to load from a constant.  */
  if (CONSTANT_P (x)
      && (MAYBE_MMX_CLASS_P (regclass)
	  || MAYBE_SSE_CLASS_P (regclass)
	  || MAYBE_MASK_CLASS_P (regclass)))
    return NO_REGS;

  /* Floating-point constants need more complex checks.  */
  if (CONST_DOUBLE_P (x))
    {
      /* General regs can load everything.  */
      if (INTEGER_CLASS_P (regclass))
        return regclass;

      /* Floats can load 0 and 1 plus some others.  Note that we eliminated
	 zero above.  We only want to wind up preferring 80387 registers if
	 we plan on doing computation with them.  */
      if (IS_STACK_MODE (mode)
	  && standard_80387_constant_p (x) > 0)
	{
	  /* Limit class to FP regs.  */
	  if (FLOAT_CLASS_P (regclass))
	    return FLOAT_REGS;
	}

      return NO_REGS;
    }

  /* Prefer SSE if we can use them for math.  Also allow integer regs
     when moves between register units are cheap.  */
  if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
    {
      if (TARGET_INTER_UNIT_MOVES_FROM_VEC
	  && TARGET_INTER_UNIT_MOVES_TO_VEC
	  && GET_MODE_SIZE (mode) <= GET_MODE_SIZE (word_mode))
	return INT_SSE_CLASS_P (regclass) ? regclass : NO_REGS;
      else
	return SSE_CLASS_P (regclass) ? regclass : NO_REGS;
    }

  /* Generally when we see PLUS here, it's the function invariant
     (plus soft-fp const_int).  Which can only be computed into general
     regs.  */
  if (GET_CODE (x) == PLUS)
    return INTEGER_CLASS_P (regclass) ? regclass : NO_REGS;

  /* QImode constants are easy to load, but non-constant QImode data
     must go into Q_REGS or ALL_MASK_REGS.  */
  if (GET_MODE (x) == QImode && !CONSTANT_P (x))
    {
      if (Q_CLASS_P (regclass))
	return regclass;
      else if (reg_class_subset_p (Q_REGS, regclass))
	return Q_REGS;
      else if (MASK_CLASS_P (regclass))
	return regclass;
      else
	return NO_REGS;
    }

  return regclass;
}

/* Discourage putting floating-point values in SSE registers unless
   SSE math is being used, and likewise for the 387 registers.  */
static reg_class_t
ix86_preferred_output_reload_class (rtx x, reg_class_t regclass)
{
  /* Restrict the output reload class to the register bank that we are doing
     math on.  If we would like not to return a subset of CLASS, reject this
     alternative: if reload cannot do this, it will still use its choice.  */
  machine_mode mode = GET_MODE (x);
  if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
    return MAYBE_SSE_CLASS_P (regclass) ? ALL_SSE_REGS : NO_REGS;

  if (IS_STACK_MODE (mode))
    return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS;

  return regclass;
}

static reg_class_t
ix86_secondary_reload (bool in_p, rtx x, reg_class_t rclass,
		       machine_mode mode, secondary_reload_info *sri)
{
  /* Double-word spills from general registers to non-offsettable memory
     references (zero-extended addresses) require special handling.  */
  if (TARGET_64BIT
      && MEM_P (x)
      && GET_MODE_SIZE (mode) > UNITS_PER_WORD
      && INTEGER_CLASS_P (rclass)
      && !offsettable_memref_p (x))
    {
      sri->icode = (in_p
		    ? CODE_FOR_reload_noff_load
		    : CODE_FOR_reload_noff_store);
      /* Add the cost of moving address to a temporary.  */
      sri->extra_cost = 1;

      return NO_REGS;
    }

  /* QImode spills from non-QI registers require
     intermediate register on 32bit targets.  */
  if (mode == QImode
      && ((!TARGET_64BIT && !in_p
	   && INTEGER_CLASS_P (rclass)
	   && MAYBE_NON_Q_CLASS_P (rclass))
	  || (!TARGET_AVX512DQ
	      && MAYBE_MASK_CLASS_P (rclass))))
    {
      int regno = true_regnum (x);

      /* Return Q_REGS if the operand is in memory.  */
      if (regno == -1)
	return Q_REGS;

      return NO_REGS;
    }

  /* Require movement to gpr, and then store to memory.  */
  if ((mode == HFmode || mode == HImode || mode == V2QImode
       || mode == BFmode)
      && !TARGET_SSE4_1
      && SSE_CLASS_P (rclass)
      && !in_p && MEM_P (x))
    {
      sri->extra_cost = 1;
      return GENERAL_REGS;
    }

  /* This condition handles corner case where an expression involving
     pointers gets vectorized.  We're trying to use the address of a
     stack slot as a vector initializer.

     (set (reg:V2DI 74 [ vect_cst_.2 ])
          (vec_duplicate:V2DI (reg/f:DI 20 frame)))

     Eventually frame gets turned into sp+offset like this:

     (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
          (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
	                               (const_int 392 [0x188]))))

     That later gets turned into:

     (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
          (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
	    (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))

     We'll have the following reload recorded:

     Reload 0: reload_in (DI) =
           (plus:DI (reg/f:DI 7 sp)
            (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
     reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
     SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
     reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
     reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
     reload_reg_rtx: (reg:V2DI 22 xmm1)

     Which isn't going to work since SSE instructions can't handle scalar
     additions.  Returning GENERAL_REGS forces the addition into integer
     register and reload can handle subsequent reloads without problems.  */

  if (in_p && GET_CODE (x) == PLUS
      && SSE_CLASS_P (rclass)
      && SCALAR_INT_MODE_P (mode))
    return GENERAL_REGS;

  return NO_REGS;
}

/* Implement TARGET_CLASS_LIKELY_SPILLED_P.  */

static bool
ix86_class_likely_spilled_p (reg_class_t rclass)
{
  switch (rclass)
    {
      case AREG:
      case DREG:
      case CREG:
      case BREG:
      case AD_REGS:
      case SIREG:
      case DIREG:
      case SSE_FIRST_REG:
      case FP_TOP_REG:
      case FP_SECOND_REG:
	return true;

      default:
	break;
    }

  return false;
}

/* Return true if a set of DST by the expression SRC should be allowed.
   This prevents complex sets of likely_spilled hard regs before reload.  */

bool
ix86_hardreg_mov_ok (rtx dst, rtx src)
{
  /* Avoid complex sets of likely_spilled hard registers before reload.  */
  if (REG_P (dst) && HARD_REGISTER_P (dst)
      && !REG_P (src) && !MEM_P (src)
      && !(VECTOR_MODE_P (GET_MODE (dst))
	   ? standard_sse_constant_p (src, GET_MODE (dst))
	   : x86_64_immediate_operand (src, GET_MODE (dst)))
      && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (dst)))
      && !reload_completed)
    return false;
  return true;
}

/* If we are copying between registers from different register sets
   (e.g. FP and integer), we may need a memory location.

   The function can't work reliably when one of the CLASSES is a class
   containing registers from multiple sets.  We avoid this by never combining
   different sets in a single alternative in the machine description.
   Ensure that this constraint holds to avoid unexpected surprises.

   When STRICT is false, we are being called from REGISTER_MOVE_COST,
   so do not enforce these sanity checks.

   To optimize register_move_cost performance, define inline variant.  */

static inline bool
inline_secondary_memory_needed (machine_mode mode, reg_class_t class1,
				reg_class_t class2, int strict)
{
  if (lra_in_progress && (class1 == NO_REGS || class2 == NO_REGS))
    return false;

  if (MAYBE_FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class1)
      || MAYBE_FLOAT_CLASS_P (class2) != FLOAT_CLASS_P (class2)
      || MAYBE_SSE_CLASS_P (class1) != SSE_CLASS_P (class1)
      || MAYBE_SSE_CLASS_P (class2) != SSE_CLASS_P (class2)
      || MAYBE_MMX_CLASS_P (class1) != MMX_CLASS_P (class1)
      || MAYBE_MMX_CLASS_P (class2) != MMX_CLASS_P (class2)
      || MAYBE_MASK_CLASS_P (class1) != MASK_CLASS_P (class1)
      || MAYBE_MASK_CLASS_P (class2) != MASK_CLASS_P (class2))
    {
      gcc_assert (!strict || lra_in_progress);
      return true;
    }

  if (FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class2))
    return true;

  /* ??? This is a lie.  We do have moves between mmx/general, and for
     mmx/sse2.  But by saying we need secondary memory we discourage the
     register allocator from using the mmx registers unless needed.  */
  if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
    return true;

  /* Between mask and general, we have moves no larger than word size.  */
  if (MASK_CLASS_P (class1) != MASK_CLASS_P (class2))
    {
      if (!(INTEGER_CLASS_P (class1) || INTEGER_CLASS_P (class2))
	  || GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	return true;
    }

  if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
    {
      /* SSE1 doesn't have any direct moves from other classes.  */
      if (!TARGET_SSE2)
	return true;

      if (!(INTEGER_CLASS_P (class1) || INTEGER_CLASS_P (class2)))
	return true;

      int msize = GET_MODE_SIZE (mode);

      /* Between SSE and general, we have moves no larger than word size.  */
      if (msize > UNITS_PER_WORD)
	return true;

      /* In addition to SImode moves, HImode moves are supported for SSE2 and above,
	 Use vmovw with AVX512FP16, or pinsrw/pextrw without AVX512FP16.  */
      int minsize = GET_MODE_SIZE (TARGET_SSE2 ? HImode : SImode);

      if (msize < minsize)
	return true;

      /* If the target says that inter-unit moves are more expensive
	 than moving through memory, then don't generate them.  */
      if ((SSE_CLASS_P (class1) && !TARGET_INTER_UNIT_MOVES_FROM_VEC)
	  || (SSE_CLASS_P (class2) && !TARGET_INTER_UNIT_MOVES_TO_VEC))
	return true;
    }

  return false;
}

/* Implement TARGET_SECONDARY_MEMORY_NEEDED.  */

static bool
ix86_secondary_memory_needed (machine_mode mode, reg_class_t class1,
			      reg_class_t class2)
{
  return inline_secondary_memory_needed (mode, class1, class2, true);
}

/* Implement TARGET_SECONDARY_MEMORY_NEEDED_MODE.

   get_secondary_mem widens integral modes to BITS_PER_WORD.
   There is no need to emit full 64 bit move on 64 bit targets
   for integral modes that can be moved using 32 bit move.  */

static machine_mode
ix86_secondary_memory_needed_mode (machine_mode mode)
{
  if (GET_MODE_BITSIZE (mode) < 32 && INTEGRAL_MODE_P (mode))
    return mode_for_size (32, GET_MODE_CLASS (mode), 0).require ();
  return mode;
}

/* Implement the TARGET_CLASS_MAX_NREGS hook.

   On the 80386, this is the size of MODE in words,
   except in the FP regs, where a single reg is always enough.  */

static unsigned char
ix86_class_max_nregs (reg_class_t rclass, machine_mode mode)
{
  if (MAYBE_INTEGER_CLASS_P (rclass))
    {
      if (mode == XFmode)
	return (TARGET_64BIT ? 2 : 3);
      else if (mode == XCmode)
	return (TARGET_64BIT ? 4 : 6);
      else
	return CEIL (GET_MODE_SIZE (mode), UNITS_PER_WORD);
    }
  else
    {
      if (COMPLEX_MODE_P (mode))
	return 2;
      else
	return 1;
    }
}

/* Implement TARGET_CAN_CHANGE_MODE_CLASS.  */

static bool
ix86_can_change_mode_class (machine_mode from, machine_mode to,
			    reg_class_t regclass)
{
  if (from == to)
    return true;

  /* x87 registers can't do subreg at all, as all values are reformatted
     to extended precision.  */
  if (MAYBE_FLOAT_CLASS_P (regclass))
    return false;

  if (MAYBE_SSE_CLASS_P (regclass) || MAYBE_MMX_CLASS_P (regclass))
    {
      /* Vector registers do not support QI or HImode loads.  If we don't
	 disallow a change to these modes, reload will assume it's ok to
	 drop the subreg from (subreg:SI (reg:HI 100) 0).  This affects
	 the vec_dupv4hi pattern.
	 NB: SSE2 can load 16bit data to sse register via pinsrw.  */
      int mov_size = MAYBE_SSE_CLASS_P (regclass) && TARGET_SSE2 ? 2 : 4;
      if (GET_MODE_SIZE (from) < mov_size
	  || GET_MODE_SIZE (to) < mov_size)
	return false;
    }

  return true;
}

/* Return index of MODE in the sse load/store tables.  */

static inline int
sse_store_index (machine_mode mode)
{
  /* NB: Use SFmode cost for HFmode instead of adding HFmode load/store
     costs to processor_costs, which requires changes to all entries in
     processor cost table.  */
  if (mode == E_HFmode)
    mode = E_SFmode;

  switch (GET_MODE_SIZE (mode))
    {
    case 4:
      return 0;
    case 8:
      return 1;
    case 16:
      return 2;
    case 32:
      return 3;
    case 64:
      return 4;
    default:
      return -1;
    }
}

/* Return the cost of moving data of mode M between a
   register and memory.  A value of 2 is the default; this cost is
   relative to those in `REGISTER_MOVE_COST'.

   This function is used extensively by register_move_cost that is used to
   build tables at startup.  Make it inline in this case.
   When IN is 2, return maximum of in and out move cost.

   If moving between registers and memory is more expensive than
   between two registers, you should define this macro to express the
   relative cost.

   Model also increased moving costs of QImode registers in non
   Q_REGS classes.
 */
static inline int
inline_memory_move_cost (machine_mode mode, enum reg_class regclass, int in)
{
  int cost;

  if (FLOAT_CLASS_P (regclass))
    {
      int index;
      switch (mode)
	{
	  case E_SFmode:
	    index = 0;
	    break;
	  case E_DFmode:
	    index = 1;
	    break;
	  case E_XFmode:
	    index = 2;
	    break;
	  default:
	    return 100;
	}
      if (in == 2)
        return MAX (ix86_cost->hard_register.fp_load [index],
		    ix86_cost->hard_register.fp_store [index]);
      return in ? ix86_cost->hard_register.fp_load [index]
		: ix86_cost->hard_register.fp_store [index];
    }
  if (SSE_CLASS_P (regclass))
    {
      int index = sse_store_index (mode);
      if (index == -1)
	return 100;
      if (in == 2)
        return MAX (ix86_cost->hard_register.sse_load [index],
		    ix86_cost->hard_register.sse_store [index]);
      return in ? ix86_cost->hard_register.sse_load [index]
		: ix86_cost->hard_register.sse_store [index];
    }
  if (MASK_CLASS_P (regclass))
    {
      int index;
      switch (GET_MODE_SIZE (mode))
	{
	case 1:
	  index = 0;
	  break;
	case 2:
	  index = 1;
	  break;
	/* DImode loads and stores assumed to cost the same as SImode.  */
	case 4:
	case 8:
	  index = 2;
	  break;
	default:
	  return 100;
	}

      if (in == 2)
	return MAX (ix86_cost->hard_register.mask_load[index],
		    ix86_cost->hard_register.mask_store[index]);
      return in ? ix86_cost->hard_register.mask_load[2]
		: ix86_cost->hard_register.mask_store[2];
    }
  if (MMX_CLASS_P (regclass))
    {
      int index;
      switch (GET_MODE_SIZE (mode))
	{
	  case 4:
	    index = 0;
	    break;
	  case 8:
	    index = 1;
	    break;
	  default:
	    return 100;
	}
      if (in == 2)
        return MAX (ix86_cost->hard_register.mmx_load [index],
		    ix86_cost->hard_register.mmx_store [index]);
      return in ? ix86_cost->hard_register.mmx_load [index]
		: ix86_cost->hard_register.mmx_store [index];
    }
  switch (GET_MODE_SIZE (mode))
    {
      case 1:
	if (Q_CLASS_P (regclass) || TARGET_64BIT)
	  {
	    if (!in)
	      return ix86_cost->hard_register.int_store[0];
	    if (TARGET_PARTIAL_REG_DEPENDENCY
	        && optimize_function_for_speed_p (cfun))
	      cost = ix86_cost->hard_register.movzbl_load;
	    else
	      cost = ix86_cost->hard_register.int_load[0];
	    if (in == 2)
	      return MAX (cost, ix86_cost->hard_register.int_store[0]);
	    return cost;
	  }
	else
	  {
	   if (in == 2)
	     return MAX (ix86_cost->hard_register.movzbl_load,
			 ix86_cost->hard_register.int_store[0] + 4);
	   if (in)
	     return ix86_cost->hard_register.movzbl_load;
	   else
	     return ix86_cost->hard_register.int_store[0] + 4;
	  }
	break;
      case 2:
	{
	  int cost;
	  if (in == 2)
	    cost = MAX (ix86_cost->hard_register.int_load[1],
			ix86_cost->hard_register.int_store[1]);
	  else
	    cost = in ? ix86_cost->hard_register.int_load[1]
		      : ix86_cost->hard_register.int_store[1];

	  if (mode == E_HFmode)
	    {
	      /* Prefer SSE over GPR for HFmode.  */
	      int sse_cost;
	      int index = sse_store_index (mode);
	      if (in == 2)
		sse_cost = MAX (ix86_cost->hard_register.sse_load[index],
				ix86_cost->hard_register.sse_store[index]);
	      else
		sse_cost = (in
			    ? ix86_cost->hard_register.sse_load [index]
			    : ix86_cost->hard_register.sse_store [index]);
	      if (sse_cost >= cost)
		cost = sse_cost + 1;
	    }
	  return cost;
	}
      default:
	if (in == 2)
	  cost = MAX (ix86_cost->hard_register.int_load[2],
		      ix86_cost->hard_register.int_store[2]);
	else if (in)
	  cost = ix86_cost->hard_register.int_load[2];
	else
	  cost = ix86_cost->hard_register.int_store[2];
	/* Multiply with the number of GPR moves needed.  */
	return cost * CEIL ((int) GET_MODE_SIZE (mode), UNITS_PER_WORD);
    }
}

static int
ix86_memory_move_cost (machine_mode mode, reg_class_t regclass, bool in)
{
  return inline_memory_move_cost (mode, (enum reg_class) regclass, in ? 1 : 0);
}


/* Return the cost of moving data from a register in class CLASS1 to
   one in class CLASS2.

   It is not required that the cost always equal 2 when FROM is the same as TO;
   on some machines it is expensive to move between registers if they are not
   general registers.  */

static int
ix86_register_move_cost (machine_mode mode, reg_class_t class1_i,
			 reg_class_t class2_i)
{
  enum reg_class class1 = (enum reg_class) class1_i;
  enum reg_class class2 = (enum reg_class) class2_i;

  /* In case we require secondary memory, compute cost of the store followed
     by load.  In order to avoid bad register allocation choices, we need
     for this to be *at least* as high as the symmetric MEMORY_MOVE_COST.  */

  if (inline_secondary_memory_needed (mode, class1, class2, false))
    {
      int cost = 1;

      cost += inline_memory_move_cost (mode, class1, 2);
      cost += inline_memory_move_cost (mode, class2, 2);

      /* In case of copying from general_purpose_register we may emit multiple
         stores followed by single load causing memory size mismatch stall.
         Count this as arbitrarily high cost of 20.  */
      if (GET_MODE_BITSIZE (mode) > BITS_PER_WORD
	  && TARGET_MEMORY_MISMATCH_STALL
	  && targetm.class_max_nregs (class1, mode)
	     > targetm.class_max_nregs (class2, mode))
	cost += 20;

      /* In the case of FP/MMX moves, the registers actually overlap, and we
	 have to switch modes in order to treat them differently.  */
      if ((MMX_CLASS_P (class1) && MAYBE_FLOAT_CLASS_P (class2))
          || (MMX_CLASS_P (class2) && MAYBE_FLOAT_CLASS_P (class1)))
	cost += 20;

      return cost;
    }

  /* Moves between MMX and non-MMX units require secondary memory.  */
  if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
    gcc_unreachable ();

  if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
    return (SSE_CLASS_P (class1)
	    ? ix86_cost->hard_register.sse_to_integer
	    : ix86_cost->hard_register.integer_to_sse);

  /* Moves between mask register and GPR.  */
  if (MASK_CLASS_P (class1) != MASK_CLASS_P (class2))
    {
      return (MASK_CLASS_P (class1)
	      ? ix86_cost->hard_register.mask_to_integer
	      : ix86_cost->hard_register.integer_to_mask);
    }
  /* Moving between mask registers.  */
  if (MASK_CLASS_P (class1) && MASK_CLASS_P (class2))
    return ix86_cost->hard_register.mask_move;

  if (MAYBE_FLOAT_CLASS_P (class1))
    return ix86_cost->hard_register.fp_move;
  if (MAYBE_SSE_CLASS_P (class1))
    {
      if (GET_MODE_BITSIZE (mode) <= 128)
	return ix86_cost->hard_register.xmm_move;
      if (GET_MODE_BITSIZE (mode) <= 256)
	return ix86_cost->hard_register.ymm_move;
      return ix86_cost->hard_register.zmm_move;
    }
  if (MAYBE_MMX_CLASS_P (class1))
    return ix86_cost->hard_register.mmx_move;
  return 2;
}

/* Implement TARGET_HARD_REGNO_NREGS.  This is ordinarily the length in
   words of a value of mode MODE but can be less for certain modes in
   special long registers.

   Actually there are no two word move instructions for consecutive
   registers.  And only registers 0-3 may have mov byte instructions
   applied to them.  */

static unsigned int
ix86_hard_regno_nregs (unsigned int regno, machine_mode mode)
{
  if (GENERAL_REGNO_P (regno))
    {
      if (mode == XFmode)
	return TARGET_64BIT ? 2 : 3;
      if (mode == XCmode)
	return TARGET_64BIT ? 4 : 6;
      return CEIL (GET_MODE_SIZE (mode), UNITS_PER_WORD);
    }
  if (COMPLEX_MODE_P (mode))
    return 2;
  /* Register pair for mask registers.  */
  if (mode == P2QImode || mode == P2HImode)
    return 2;
  if (mode == V64SFmode || mode == V64SImode)
    return 4;
  return 1;
}

/* Implement REGMODE_NATURAL_SIZE(MODE).  */
unsigned int
ix86_regmode_natural_size (machine_mode mode)
{
  if (mode == P2HImode || mode == P2QImode)
    return GET_MODE_SIZE (mode) / 2;
  return UNITS_PER_WORD;
}

/* Implement TARGET_HARD_REGNO_MODE_OK.  */

static bool
ix86_hard_regno_mode_ok (unsigned int regno, machine_mode mode)
{
  /* Flags and only flags can only hold CCmode values.  */
  if (CC_REGNO_P (regno))
    return GET_MODE_CLASS (mode) == MODE_CC;
  if (GET_MODE_CLASS (mode) == MODE_CC
      || GET_MODE_CLASS (mode) == MODE_RANDOM)
    return false;
  if (STACK_REGNO_P (regno))
    return VALID_FP_MODE_P (mode);
  if (MASK_REGNO_P (regno))
    {
      /* Register pair only starts at even register number.  */
      if ((mode == P2QImode || mode == P2HImode))
	return MASK_PAIR_REGNO_P(regno);

      return ((TARGET_AVX512F && VALID_MASK_REG_MODE (mode))
	      || (TARGET_AVX512BW
		  && VALID_MASK_AVX512BW_MODE (mode)));
    }

  if (GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
    return false;

  if (SSE_REGNO_P (regno))
    {
      /* We implement the move patterns for all vector modes into and
	 out of SSE registers, even when no operation instructions
	 are available.  */

      /* For AVX-512 we allow, regardless of regno:
	  - XI mode
	  - any of 512-bit wide vector mode
	  - any scalar mode.  */
      if (TARGET_AVX512F
	  && (VALID_AVX512F_REG_OR_XI_MODE (mode)
	      || VALID_AVX512F_SCALAR_MODE (mode)))
	return true;

      /* For AVX-5124FMAPS or AVX-5124VNNIW
	 allow V64SF and V64SI modes for special regnos.  */
      if ((TARGET_AVX5124FMAPS || TARGET_AVX5124VNNIW)
	  && (mode == V64SFmode || mode == V64SImode)
	  && MOD4_SSE_REGNO_P (regno))
	return true;

      /* TODO check for QI/HI scalars.  */
      /* AVX512VL allows sse regs16+ for 128/256 bit modes.  */
      if (TARGET_AVX512VL
	  && (VALID_AVX256_REG_OR_OI_MODE (mode)
	      || VALID_AVX512VL_128_REG_MODE (mode)))
	return true;

      /* xmm16-xmm31 are only available for AVX-512.  */
      if (EXT_REX_SSE_REGNO_P (regno))
	return false;

      /* Use pinsrw/pextrw to mov 16-bit data from/to sse to/from integer.  */
      if (TARGET_SSE2 && mode == HImode)
	return true;

      /* OImode and AVX modes are available only when AVX is enabled.  */
      return ((TARGET_AVX
	       && VALID_AVX256_REG_OR_OI_MODE (mode))
	      || VALID_SSE_REG_MODE (mode)
	      || VALID_SSE2_REG_MODE (mode)
	      || VALID_MMX_REG_MODE (mode)
	      || VALID_MMX_REG_MODE_3DNOW (mode));
    }
  if (MMX_REGNO_P (regno))
    {
      /* We implement the move patterns for 3DNOW modes even in MMX mode,
	 so if the register is available at all, then we can move data of
	 the given mode into or out of it.  */
      return (VALID_MMX_REG_MODE (mode)
	      || VALID_MMX_REG_MODE_3DNOW (mode));
    }

  if (mode == QImode)
    {
      /* Take care for QImode values - they can be in non-QI regs,
	 but then they do cause partial register stalls.  */
      if (ANY_QI_REGNO_P (regno))
	return true;
      if (!TARGET_PARTIAL_REG_STALL)
	return true;
      /* LRA checks if the hard register is OK for the given mode.
	 QImode values can live in non-QI regs, so we allow all
	 registers here.  */
      if (lra_in_progress)
       return true;
      return !can_create_pseudo_p ();
    }
  /* We handle both integer and floats in the general purpose registers.  */
  else if (VALID_INT_MODE_P (mode)
	   || VALID_FP_MODE_P (mode))
    return true;
  /* Lots of MMX code casts 8 byte vector modes to DImode.  If we then go
     on to use that value in smaller contexts, this can easily force a
     pseudo to be allocated to GENERAL_REGS.  Since this is no worse than
     supporting DImode, allow it.  */
  else if (VALID_MMX_REG_MODE_3DNOW (mode) || VALID_MMX_REG_MODE (mode))
    return true;

  return false;
}

/* Implement TARGET_INSN_CALLEE_ABI.  */

const predefined_function_abi &
ix86_insn_callee_abi (const rtx_insn *insn)
{
  unsigned int abi_id = 0;
  rtx pat = PATTERN (insn);
  if (vzeroupper_pattern (pat, VOIDmode))
    abi_id = ABI_VZEROUPPER;

  return function_abis[abi_id];
}

/* Initialize function_abis with corresponding abi_id,
   currently only handle vzeroupper.  */
void
ix86_initialize_callee_abi (unsigned int abi_id)
{
  gcc_assert (abi_id == ABI_VZEROUPPER);
  predefined_function_abi &vzeroupper_abi = function_abis[abi_id];
  if (!vzeroupper_abi.initialized_p ())
    {
      HARD_REG_SET full_reg_clobbers;
      CLEAR_HARD_REG_SET (full_reg_clobbers);
      vzeroupper_abi.initialize (ABI_VZEROUPPER, full_reg_clobbers);
    }
}

void
ix86_expand_avx_vzeroupper (void)
{
  /* Initialize vzeroupper_abi here.  */
  ix86_initialize_callee_abi (ABI_VZEROUPPER);
  rtx_insn *insn = emit_call_insn (gen_avx_vzeroupper_callee_abi ());
  /* Return false for non-local goto in can_nonlocal_goto.  */
  make_reg_eh_region_note (insn, 0, INT_MIN);
  /* Flag used for call_insn indicates it's a fake call.  */
  RTX_FLAG (insn, used) = 1;
}


/* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED.  The only ABI that
   saves SSE registers across calls is Win64 (thus no need to check the
   current ABI here), and with AVX enabled Win64 only guarantees that
   the low 16 bytes are saved.  */

static bool
ix86_hard_regno_call_part_clobbered (unsigned int abi_id, unsigned int regno,
				     machine_mode mode)
{
  /* Special ABI for vzeroupper which only clobber higher part of sse regs.  */
  if (abi_id == ABI_VZEROUPPER)
      return (GET_MODE_SIZE (mode) > 16
	      && ((TARGET_64BIT && REX_SSE_REGNO_P (regno))
		  || LEGACY_SSE_REGNO_P (regno)));

  return SSE_REGNO_P (regno) && GET_MODE_SIZE (mode) > 16;
}

/* A subroutine of ix86_modes_tieable_p.  Return true if MODE is a
   tieable integer mode.  */

static bool
ix86_tieable_integer_mode_p (machine_mode mode)
{
  switch (mode)
    {
    case E_HImode:
    case E_SImode:
      return true;

    case E_QImode:
      return TARGET_64BIT || !TARGET_PARTIAL_REG_STALL;

    case E_DImode:
      return TARGET_64BIT;

    default:
      return false;
    }
}

/* Implement TARGET_MODES_TIEABLE_P.

   Return true if MODE1 is accessible in a register that can hold MODE2
   without copying.  That is, all register classes that can hold MODE2
   can also hold MODE1.  */

static bool
ix86_modes_tieable_p (machine_mode mode1, machine_mode mode2)
{
  if (mode1 == mode2)
    return true;

  if (ix86_tieable_integer_mode_p (mode1)
      && ix86_tieable_integer_mode_p (mode2))
    return true;

  /* MODE2 being XFmode implies fp stack or general regs, which means we
     can tie any smaller floating point modes to it.  Note that we do not
     tie this with TFmode.  */
  if (mode2 == XFmode)
    return mode1 == SFmode || mode1 == DFmode;

  /* MODE2 being DFmode implies fp stack, general or sse regs, which means
     that we can tie it with SFmode.  */
  if (mode2 == DFmode)
    return mode1 == SFmode;

  /* If MODE2 is only appropriate for an SSE register, then tie with
     any other mode acceptable to SSE registers.  */
  if (GET_MODE_SIZE (mode2) == 64
      && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
    return (GET_MODE_SIZE (mode1) == 64
	    && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
  if (GET_MODE_SIZE (mode2) == 32
      && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
    return (GET_MODE_SIZE (mode1) == 32
	    && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
  if (GET_MODE_SIZE (mode2) == 16
      && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
    return (GET_MODE_SIZE (mode1) == 16
	    && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));

  /* If MODE2 is appropriate for an MMX register, then tie
     with any other mode acceptable to MMX registers.  */
  if (GET_MODE_SIZE (mode2) == 8
      && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode2))
    return (GET_MODE_SIZE (mode1) == 8
	    && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode1));

  /* SCmode and DImode can be tied.  */
  if ((mode1 == E_SCmode && mode2 == E_DImode)
      || (mode1 == E_DImode && mode2 == E_SCmode))
    return TARGET_64BIT;

  /* [SD]Cmode and V2[SD]Fmode modes can be tied.  */
  if ((mode1 == E_SCmode && mode2 == E_V2SFmode)
      || (mode1 == E_V2SFmode && mode2 == E_SCmode)
      || (mode1 == E_DCmode && mode2 == E_V2DFmode)
      || (mode1 == E_V2DFmode && mode2 == E_DCmode))
    return true;

  return false;
}

/* Return the cost of moving between two registers of mode MODE.  */

static int
ix86_set_reg_reg_cost (machine_mode mode)
{
  unsigned int units = UNITS_PER_WORD;

  switch (GET_MODE_CLASS (mode))
    {
    default:
      break;

    case MODE_CC:
      units = GET_MODE_SIZE (CCmode);
      break;

    case MODE_FLOAT:
      if ((TARGET_SSE && mode == TFmode)
	  || (TARGET_80387 && mode == XFmode)
	  || ((TARGET_80387 || TARGET_SSE2) && mode == DFmode)
	  || ((TARGET_80387 || TARGET_SSE) && mode == SFmode))
	units = GET_MODE_SIZE (mode);
      break;

    case MODE_COMPLEX_FLOAT:
      if ((TARGET_SSE && mode == TCmode)
	  || (TARGET_80387 && mode == XCmode)
	  || ((TARGET_80387 || TARGET_SSE2) && mode == DCmode)
	  || ((TARGET_80387 || TARGET_SSE) && mode == SCmode))
	units = GET_MODE_SIZE (mode);
      break;

    case MODE_VECTOR_INT:
    case MODE_VECTOR_FLOAT:
      if ((TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
	  || (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
	  || (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
	  || (TARGET_SSE && VALID_SSE_REG_MODE (mode))
	  || ((TARGET_MMX || TARGET_MMX_WITH_SSE)
	      && VALID_MMX_REG_MODE (mode)))
	units = GET_MODE_SIZE (mode);
    }

  /* Return the cost of moving between two registers of mode MODE,
     assuming that the move will be in pieces of at most UNITS bytes.  */
  return COSTS_N_INSNS (CEIL (GET_MODE_SIZE (mode), units));
}

/* Return cost of vector operation in MODE given that scalar version has
   COST.  */

static int
ix86_vec_cost (machine_mode mode, int cost)
{
  if (!VECTOR_MODE_P (mode))
    return cost;

  if (GET_MODE_BITSIZE (mode) == 128
      && TARGET_SSE_SPLIT_REGS)
    return cost * GET_MODE_BITSIZE (mode) / 64;
  else if (GET_MODE_BITSIZE (mode) > 128
      && TARGET_AVX256_SPLIT_REGS)
    return cost * GET_MODE_BITSIZE (mode) / 128;
  else if (GET_MODE_BITSIZE (mode) > 256
      && TARGET_AVX512_SPLIT_REGS)
    return cost * GET_MODE_BITSIZE (mode) / 256;
  return cost;
}

/* Return cost of vec_widen_<s>mult_hi/lo_<mode>,
   vec_widen_<s>mul_hi/lo_<mode> is only available for VI124_AVX2.  */
static int
ix86_widen_mult_cost (const struct processor_costs *cost,
		      enum machine_mode mode, bool uns_p)
{
  gcc_assert (GET_MODE_CLASS (mode) == MODE_VECTOR_INT);
  int extra_cost = 0;
  int basic_cost = 0;
  switch (mode)
    {
    case V8HImode:
    case V16HImode:
      if (!uns_p || mode == V16HImode)
	extra_cost = cost->sse_op * 2;
      basic_cost = cost->mulss * 2 + cost->sse_op * 4;
      break;
    case V4SImode:
    case V8SImode:
      /* pmulhw/pmullw can be used.  */
      basic_cost = cost->mulss * 2 + cost->sse_op * 2;
      break;
    case V2DImode:
      /* pmuludq under sse2, pmuldq under sse4.1, for sign_extend,
	 require extra 4 mul, 4 add, 4 cmp and 2 shift.  */
      if (!TARGET_SSE4_1 && !uns_p)
	extra_cost = (cost->mulss + cost->addss + cost->sse_op) * 4
		      + cost->sse_op * 2;
      /* Fallthru.  */
    case V4DImode:
      basic_cost = cost->mulss * 2 + cost->sse_op * 4;
      break;
    default:
      /* Not implemented.  */
      return 100;
    }
  return ix86_vec_cost (mode, basic_cost + extra_cost);
}

/* Return cost of multiplication in MODE.  */

static int
ix86_multiplication_cost (const struct processor_costs *cost,
			  enum machine_mode mode)
{
  machine_mode inner_mode = mode;
  if (VECTOR_MODE_P (mode))
    inner_mode = GET_MODE_INNER (mode);

  if (SSE_FLOAT_MODE_SSEMATH_OR_HF_P (mode))
    return inner_mode == DFmode ? cost->mulsd : cost->mulss;
  else if (X87_FLOAT_MODE_P (mode))
    return cost->fmul;
  else if (FLOAT_MODE_P (mode))
    return  ix86_vec_cost (mode,
			   inner_mode == DFmode ? cost->mulsd : cost->mulss);
  else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
    {
      int nmults, nops;
      /* Cost of reading the memory.  */
      int extra;

      switch (mode)
	{
	case V4QImode:
	case V8QImode:
	  /* Partial V*QImode is emulated with 4-6 insns.  */
	  nmults = 1;
	  nops = 3;
	  extra = 0;

	  if (TARGET_AVX512BW && TARGET_AVX512VL)
	    ;
	  else if (TARGET_AVX2)
	    nops += 2;
	  else if (TARGET_XOP)
	    extra += cost->sse_load[2];
	  else
	    {
	      nops += 1;
	      extra += cost->sse_load[2];
	    }
	  goto do_qimode;

	case V16QImode:
	  /* V*QImode is emulated with 4-11 insns.  */
	  nmults = 1;
	  nops = 3;
	  extra = 0;

	  if (TARGET_AVX2 && !TARGET_PREFER_AVX128)
	    {
	      if (!(TARGET_AVX512BW && TARGET_AVX512VL))
		nops += 3;
	    }
	  else if (TARGET_XOP)
	    {
	      nmults += 1;
	      nops += 2;
	      extra += cost->sse_load[2];
	    }
	  else
	    {
	      nmults += 1;
	      nops += 4;
	      extra += cost->sse_load[2];
	    }
	  goto do_qimode;

	case V32QImode:
	  nmults = 1;
	  nops = 3;
	  extra = 0;

	  if (!TARGET_AVX512BW || TARGET_PREFER_AVX256)
	    {
	      nmults += 1;
	      nops += 4;
	      extra += cost->sse_load[3] * 2;
	    }
	  goto do_qimode;

	case V64QImode:
	  nmults = 2;
	  nops = 9;
	  extra = cost->sse_load[3] * 2 + cost->sse_load[4] * 2;

	do_qimode:
	  return ix86_vec_cost (mode, cost->mulss * nmults
				+ cost->sse_op * nops) + extra;

	case V4SImode:
	  /* pmulld is used in this case. No emulation is needed.  */
	  if (TARGET_SSE4_1)
	    goto do_native;
	  /* V4SImode is emulated with 7 insns.  */
	  else
	    return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 5);

	case V2DImode:
	case V4DImode:
	  /* vpmullq is used in this case. No emulation is needed.  */
	  if (TARGET_AVX512DQ && TARGET_AVX512VL)
	    goto do_native;
	  /* V*DImode is emulated with 6-8 insns.  */
	  else if (TARGET_XOP && mode == V2DImode)
	    return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 4);
	  /* FALLTHRU */
	case V8DImode:
	  /* vpmullq is used in this case. No emulation is needed.  */
	  if (TARGET_AVX512DQ && mode == V8DImode)
	    goto do_native;
	  else
	    return ix86_vec_cost (mode, cost->mulss * 3 + cost->sse_op * 5);

	default:
	do_native:
	  return ix86_vec_cost (mode, cost->mulss);
	}
    }
  else
    return (cost->mult_init[MODE_INDEX (mode)] + cost->mult_bit * 7);
}

/* Return cost of multiplication in MODE.  */

static int
ix86_division_cost (const struct processor_costs *cost,
			  enum machine_mode mode)
{
  machine_mode inner_mode = mode;
  if (VECTOR_MODE_P (mode))
    inner_mode = GET_MODE_INNER (mode);

  if (SSE_FLOAT_MODE_SSEMATH_OR_HF_P (mode))
    return inner_mode == DFmode ? cost->divsd : cost->divss;
  else if (X87_FLOAT_MODE_P (mode))
    return cost->fdiv;
  else if (FLOAT_MODE_P (mode))
    return ix86_vec_cost (mode,
			  inner_mode == DFmode ? cost->divsd : cost->divss);
  else
    return cost->divide[MODE_INDEX (mode)];
}

/* Return cost of shift in MODE.
   If CONSTANT_OP1 is true, the op1 value is known and set in OP1_VAL.
   AND_IN_OP1 specify in op1 is result of AND and SHIFT_AND_TRUNCATE
   if op1 is a result of subreg.

   SKIP_OP0/1 is set to true if cost of OP0/1 should be ignored.  */

static int
ix86_shift_rotate_cost (const struct processor_costs *cost,
			enum rtx_code code,
			enum machine_mode mode, bool constant_op1,
			HOST_WIDE_INT op1_val,
			bool and_in_op1,
			bool shift_and_truncate,
			bool *skip_op0, bool *skip_op1)
{
  if (skip_op0)
    *skip_op0 = *skip_op1 = false;

  if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
    {
      int count;
      /* Cost of reading the memory.  */
      int extra;

      switch (mode)
	{
	case V4QImode:
	case V8QImode:
	  if (TARGET_AVX2)
	    /* Use vpbroadcast.  */
	    extra = cost->sse_op;
	  else
	    extra = cost->sse_load[2];

	  if (constant_op1)
	    {
	      if (code == ASHIFTRT)
		{
		  count = 4;
		  extra *= 2;
		}
	      else
		count = 2;
	    }
	  else if (TARGET_AVX512BW && TARGET_AVX512VL)
	    return ix86_vec_cost (mode, cost->sse_op * 4);
	  else if (TARGET_SSE4_1)
	    count = 5;
	  else if (code == ASHIFTRT)
	    count = 6;
	  else
	    count = 5;
	  return ix86_vec_cost (mode, cost->sse_op * count) + extra;

	case V16QImode:
	  if (TARGET_XOP)
	    {
	      /* For XOP we use vpshab, which requires a broadcast of the
		 value to the variable shift insn.  For constants this
		 means a V16Q const in mem; even when we can perform the
		 shift with one insn set the cost to prefer paddb.  */
	      if (constant_op1)
		{
		  extra = cost->sse_load[2];
		  return ix86_vec_cost (mode, cost->sse_op) + extra;
		}
	      else
		{
		  count = (code == ASHIFT) ? 3 : 4;
		  return ix86_vec_cost (mode, cost->sse_op * count);
		}
	    }
	  /* FALLTHRU */
	case V32QImode:
	  if (TARGET_AVX2)
	    /* Use vpbroadcast.  */
	    extra = cost->sse_op;
	  else
	    extra = (mode == V16QImode) ? cost->sse_load[2] : cost->sse_load[3];

	  if (constant_op1)
	    {
	      if (code == ASHIFTRT)
		{
		  count = 4;
		  extra *= 2;
		}
	      else
		count = 2;
	    }
	  else if (TARGET_AVX512BW
		   && ((mode == V32QImode && !TARGET_PREFER_AVX256)
		       || (mode == V16QImode && TARGET_AVX512VL
			   && !TARGET_PREFER_AVX128)))
	    return ix86_vec_cost (mode, cost->sse_op * 4);
	  else if (TARGET_AVX2
		   && mode == V16QImode && !TARGET_PREFER_AVX128)
	    count = 6;
	  else if (TARGET_SSE4_1)
	    count = 9;
	  else if (code == ASHIFTRT)
	    count = 10;
	  else
	    count = 9;
	  return ix86_vec_cost (mode, cost->sse_op * count) + extra;

	case V2DImode:
	case V4DImode:
	  /* V*DImode arithmetic right shift is emulated.  */
	  if (code == ASHIFTRT && !TARGET_AVX512VL)
	    {
	      if (constant_op1)
		{
		  if (op1_val == 63)
		    count = TARGET_SSE4_2 ? 1 : 2;
		  else if (TARGET_XOP)
		    count = 2;
		  else if (TARGET_SSE4_1)
		    count = 3;
		  else
		    count = 4;
		}
	      else if (TARGET_XOP)
		count = 3;
	      else if (TARGET_SSE4_2)
		count = 4;
	      else
		count = 5;

	      return ix86_vec_cost (mode, cost->sse_op * count);
	    }
	  /* FALLTHRU */
	default:
	  return ix86_vec_cost (mode, cost->sse_op);
	}
    }

  if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
    {
      if (constant_op1)
	{
	  if (op1_val > 32)
	    return cost->shift_const + COSTS_N_INSNS (2);
	  else
	    return cost->shift_const * 2;
	}
      else
	{
	  if (and_in_op1)
	    return cost->shift_var * 2;
	  else
	    return cost->shift_var * 6 + COSTS_N_INSNS (2);
	}
    }
  else
    {
      if (constant_op1)
	return cost->shift_const;
      else if (shift_and_truncate)
	{
	  if (skip_op0)
	    *skip_op0 = *skip_op1 = true;
	  /* Return the cost after shift-and truncation.  */
	  return cost->shift_var;
	}
      else
	return cost->shift_var;
    }
}

/* Compute a (partial) cost for rtx X.  Return true if the complete
   cost has been computed, and false if subexpressions should be
   scanned.  In either case, *TOTAL contains the cost result.  */

static bool
ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
		int *total, bool speed)
{
  rtx mask;
  enum rtx_code code = GET_CODE (x);
  enum rtx_code outer_code = (enum rtx_code) outer_code_i;
  const struct processor_costs *cost
    = speed ? ix86_tune_cost : &ix86_size_cost;
  int src_cost;

  switch (code)
    {
    case SET:
      if (register_operand (SET_DEST (x), VOIDmode)
	  && register_operand (SET_SRC (x), VOIDmode))
	{
	  *total = ix86_set_reg_reg_cost (GET_MODE (SET_DEST (x)));
	  return true;
	}

      if (register_operand (SET_SRC (x), VOIDmode))
	/* Avoid potentially incorrect high cost from rtx_costs
	   for non-tieable SUBREGs.  */
	src_cost = 0;
      else
	{
	  src_cost = rtx_cost (SET_SRC (x), mode, SET, 1, speed);

	  if (CONSTANT_P (SET_SRC (x)))
	    /* Constant costs assume a base value of COSTS_N_INSNS (1) and add
	       a small value, possibly zero for cheap constants.  */
	    src_cost += COSTS_N_INSNS (1);
	}

      *total = src_cost + rtx_cost (SET_DEST (x), mode, SET, 0, speed);
      return true;

    case CONST_INT:
    case CONST:
    case LABEL_REF:
    case SYMBOL_REF:
      if (x86_64_immediate_operand (x, VOIDmode))
	*total = 0;
     else
	*total = 1;
      return true;

    case CONST_DOUBLE:
      if (IS_STACK_MODE (mode))
	switch (standard_80387_constant_p (x))
	  {
	  case -1:
	  case 0:
	    break;
	  case 1: /* 0.0 */
	    *total = 1;
	    return true;
	  default: /* Other constants */
	    *total = 2;
	    return true;
	  }
      /* FALLTHRU */

    case CONST_VECTOR:
      switch (standard_sse_constant_p (x, mode))
	{
	case 0:
	  break;
	case 1:  /* 0: xor eliminates false dependency */
	  *total = 0;
	  return true;
	default: /* -1: cmp contains false dependency */
	  *total = 1;
	  return true;
	}
      /* FALLTHRU */

    case CONST_WIDE_INT:
      /* Fall back to (MEM (SYMBOL_REF)), since that's where
	 it'll probably end up.  Add a penalty for size.  */
      *total = (COSTS_N_INSNS (1)
		+ (!TARGET_64BIT && flag_pic)
		+ (GET_MODE_SIZE (mode) <= 4
		   ? 0 : GET_MODE_SIZE (mode) <= 8 ? 1 : 2));
      return true;

    case ZERO_EXTEND:
      /* The zero extensions is often completely free on x86_64, so make
	 it as cheap as possible.  */
      if (TARGET_64BIT && mode == DImode
	  && GET_MODE (XEXP (x, 0)) == SImode)
	*total = 1;
      else if (TARGET_ZERO_EXTEND_WITH_AND)
	*total = cost->add;
      else
	*total = cost->movzx;
      return false;

    case SIGN_EXTEND:
      *total = cost->movsx;
      return false;

    case ASHIFT:
      if (SCALAR_INT_MODE_P (mode)
	  && GET_MODE_SIZE (mode) < UNITS_PER_WORD
	  && CONST_INT_P (XEXP (x, 1)))
	{
	  HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
	  if (value == 1)
	    {
	      *total = cost->add;
	      return false;
	    }
	  if ((value == 2 || value == 3)
	      && cost->lea <= cost->shift_const)
	    {
	      *total = cost->lea;
	      return false;
	    }
	}
      /* FALLTHRU */

    case ROTATE:
    case ASHIFTRT:
    case LSHIFTRT:
    case ROTATERT:
      bool skip_op0, skip_op1;
      *total = ix86_shift_rotate_cost (cost, code, mode,
				       CONSTANT_P (XEXP (x, 1)),
				       CONST_INT_P (XEXP (x, 1))
					 ? INTVAL (XEXP (x, 1)) : -1,
				       GET_CODE (XEXP (x, 1)) == AND,
				       SUBREG_P (XEXP (x, 1))
				       && GET_CODE (XEXP (XEXP (x, 1),
							  0)) == AND,
				       &skip_op0, &skip_op1);
      if (skip_op0 || skip_op1)
	{
	  if (!skip_op0)
	    *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
	  if (!skip_op1)
	    *total += rtx_cost (XEXP (x, 1), mode, code, 0, speed);
	  return true;
	}
      return false;

    case FMA:
      {
	rtx sub;

        gcc_assert (FLOAT_MODE_P (mode));
        gcc_assert (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F);

        *total = ix86_vec_cost (mode,
				GET_MODE_INNER (mode) == SFmode
				? cost->fmass : cost->fmasd);
	*total += rtx_cost (XEXP (x, 1), mode, FMA, 1, speed);

        /* Negate in op0 or op2 is free: FMS, FNMA, FNMS.  */
	sub = XEXP (x, 0);
	if (GET_CODE (sub) == NEG)
	  sub = XEXP (sub, 0);
	*total += rtx_cost (sub, mode, FMA, 0, speed);

	sub = XEXP (x, 2);
	if (GET_CODE (sub) == NEG)
	  sub = XEXP (sub, 0);
	*total += rtx_cost (sub, mode, FMA, 2, speed);
	return true;
      }

    case MULT:
      if (!FLOAT_MODE_P (mode) && !VECTOR_MODE_P (mode))
	{
	  rtx op0 = XEXP (x, 0);
	  rtx op1 = XEXP (x, 1);
	  int nbits;
	  if (CONST_INT_P (XEXP (x, 1)))
	    {
	      unsigned HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
	      for (nbits = 0; value != 0; value &= value - 1)
	        nbits++;
	    }
	  else
	    /* This is arbitrary.  */
	    nbits = 7;

	  /* Compute costs correctly for widening multiplication.  */
	  if ((GET_CODE (op0) == SIGN_EXTEND || GET_CODE (op0) == ZERO_EXTEND)
	      && GET_MODE_SIZE (GET_MODE (XEXP (op0, 0))) * 2
	         == GET_MODE_SIZE (mode))
	    {
	      int is_mulwiden = 0;
	      machine_mode inner_mode = GET_MODE (op0);

	      if (GET_CODE (op0) == GET_CODE (op1))
		is_mulwiden = 1, op1 = XEXP (op1, 0);
	      else if (CONST_INT_P (op1))
		{
		  if (GET_CODE (op0) == SIGN_EXTEND)
		    is_mulwiden = trunc_int_for_mode (INTVAL (op1), inner_mode)
			          == INTVAL (op1);
		  else
		    is_mulwiden = !(INTVAL (op1) & ~GET_MODE_MASK (inner_mode));
	        }

	      if (is_mulwiden)
	        op0 = XEXP (op0, 0), mode = GET_MODE (op0);
	    }

	  int mult_init;
	  // Double word multiplication requires 3 mults and 2 adds.
	  if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	    {
	      mult_init = 3 * cost->mult_init[MODE_INDEX (word_mode)]
			  + 2 * cost->add;
	      nbits *= 3;
	    }
	  else mult_init = cost->mult_init[MODE_INDEX (mode)];

  	  *total = (mult_init
		    + nbits * cost->mult_bit
	            + rtx_cost (op0, mode, outer_code, opno, speed)
		    + rtx_cost (op1, mode, outer_code, opno, speed));

          return true;
	}
      *total = ix86_multiplication_cost (cost, mode);
      return false;

    case DIV:
    case UDIV:
    case MOD:
    case UMOD:
      *total = ix86_division_cost (cost, mode);
      return false;

    case PLUS:
      if (GET_MODE_CLASS (mode) == MODE_INT
	  && GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
	{
	  if (GET_CODE (XEXP (x, 0)) == PLUS
	      && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
	      && CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 0), 1))
	      && CONSTANT_P (XEXP (x, 1)))
	    {
	      HOST_WIDE_INT val = INTVAL (XEXP (XEXP (XEXP (x, 0), 0), 1));
	      if (val == 2 || val == 4 || val == 8)
		{
		  *total = cost->lea;
		  *total += rtx_cost (XEXP (XEXP (x, 0), 1), mode,
				      outer_code, opno, speed);
		  *total += rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0), mode,
				      outer_code, opno, speed);
		  *total += rtx_cost (XEXP (x, 1), mode,
				      outer_code, opno, speed);
		  return true;
		}
	    }
	  else if (GET_CODE (XEXP (x, 0)) == MULT
		   && CONST_INT_P (XEXP (XEXP (x, 0), 1)))
	    {
	      HOST_WIDE_INT val = INTVAL (XEXP (XEXP (x, 0), 1));
	      if (val == 2 || val == 4 || val == 8)
		{
		  *total = cost->lea;
		  *total += rtx_cost (XEXP (XEXP (x, 0), 0), mode,
				      outer_code, opno, speed);
		  *total += rtx_cost (XEXP (x, 1), mode,
				      outer_code, opno, speed);
		  return true;
		}
	    }
	  else if (GET_CODE (XEXP (x, 0)) == PLUS)
	    {
	      rtx op = XEXP (XEXP (x, 0), 0);

	      /* Add with carry, ignore the cost of adding a carry flag.  */
	      if (ix86_carry_flag_operator (op, mode)
		  || ix86_carry_flag_unset_operator (op, mode))
		*total = cost->add;
	      else
		{
		  *total = cost->lea;
		  *total += rtx_cost (op, mode,
				      outer_code, opno, speed);
		}

	      *total += rtx_cost (XEXP (XEXP (x, 0), 1), mode,
				  outer_code, opno, speed);
	      *total += rtx_cost (XEXP (x, 1), mode,
				  outer_code, opno, speed);
	      return true;
	    }
	}
      /* FALLTHRU */

    case MINUS:
      /* Subtract with borrow, ignore the cost of subtracting a carry flag.  */
      if (GET_MODE_CLASS (mode) == MODE_INT
	  && GET_MODE_SIZE (mode) <= UNITS_PER_WORD
	  && GET_CODE (XEXP (x, 0)) == MINUS
	  && (ix86_carry_flag_operator (XEXP (XEXP (x, 0), 1), mode)
	      || ix86_carry_flag_unset_operator (XEXP (XEXP (x, 0), 1), mode)))
	{
	  *total = cost->add;
	  *total += rtx_cost (XEXP (XEXP (x, 0), 0), mode,
			      outer_code, opno, speed);
	  *total += rtx_cost (XEXP (x, 1), mode,
			      outer_code, opno, speed);
	  return true;
	}

      if (SSE_FLOAT_MODE_SSEMATH_OR_HF_P (mode))
	*total = cost->addss;
      else if (X87_FLOAT_MODE_P (mode))
	*total = cost->fadd;
      else if (FLOAT_MODE_P (mode))
	*total = ix86_vec_cost (mode, cost->addss);
      else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
	*total = ix86_vec_cost (mode, cost->sse_op);
      else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	*total = cost->add * 2;
      else
	*total = cost->add;
      return false;

    case IOR:
      if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
	  || SSE_FLOAT_MODE_P (mode))
	{
	  /* (ior (not ...) ...) can be a single insn in AVX512.  */
	  if (GET_CODE (XEXP (x, 0)) == NOT && TARGET_AVX512F
	      && (GET_MODE_SIZE (mode) == 64
		  || (TARGET_AVX512VL
		      && (GET_MODE_SIZE (mode) == 32
			  || GET_MODE_SIZE (mode) == 16))))
	    {
	      rtx right = GET_CODE (XEXP (x, 1)) != NOT
			  ? XEXP (x, 1) : XEXP (XEXP (x, 1), 0);

	      *total = ix86_vec_cost (mode, cost->sse_op)
		       + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
				   outer_code, opno, speed)
		       + rtx_cost (right, mode, outer_code, opno, speed);
	      return true;
	    }
	  *total = ix86_vec_cost (mode, cost->sse_op);
	}
      else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	*total = cost->add * 2;
      else
	*total = cost->add;
      return false;

    case XOR:
      if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
	  || SSE_FLOAT_MODE_P (mode))
	*total = ix86_vec_cost (mode, cost->sse_op);
      else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	*total = cost->add * 2;
      else
	*total = cost->add;
      return false;

    case AND:
      if (address_no_seg_operand (x, mode))
	{
	  *total = cost->lea;
	  return true;
	}
      else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
	       || SSE_FLOAT_MODE_P (mode))
	{
	  /* pandn is a single instruction.  */
	  if (GET_CODE (XEXP (x, 0)) == NOT)
	    {
	      rtx right = XEXP (x, 1);

	      /* (and (not ...) (not ...)) can be a single insn in AVX512.  */
	      if (GET_CODE (right) == NOT && TARGET_AVX512F
		  && (GET_MODE_SIZE (mode) == 64
		      || (TARGET_AVX512VL
			  && (GET_MODE_SIZE (mode) == 32
			      || GET_MODE_SIZE (mode) == 16))))
		right = XEXP (right, 0);

	      *total = ix86_vec_cost (mode, cost->sse_op)
		       + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
				   outer_code, opno, speed)
		       + rtx_cost (right, mode, outer_code, opno, speed);
	      return true;
	    }
	  else if (GET_CODE (XEXP (x, 1)) == NOT)
	    {
	      *total = ix86_vec_cost (mode, cost->sse_op)
		       + rtx_cost (XEXP (x, 0), mode,
				   outer_code, opno, speed)
		       + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
				   outer_code, opno, speed);
	      return true;
	    }
	  *total = ix86_vec_cost (mode, cost->sse_op);
	}
      else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	{
	  if (TARGET_BMI && GET_CODE (XEXP (x,0)) == NOT)
	    {
	      *total = cost->add * 2
		       + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
				   outer_code, opno, speed)
		       + rtx_cost (XEXP (x, 1), mode,
				   outer_code, opno, speed);
	      return true;
	    }
	  else if (TARGET_BMI && GET_CODE (XEXP (x, 1)) == NOT)
	    {
	      *total = cost->add * 2
		       + rtx_cost (XEXP (x, 0), mode,
				   outer_code, opno, speed)
		       + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
				   outer_code, opno, speed);
	      return true;
	    }
	  *total = cost->add * 2;
	}
      else if (TARGET_BMI && GET_CODE (XEXP (x,0)) == NOT)
	{
	  *total = cost->add
		   + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
			       outer_code, opno, speed)
		   + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
	  return true;
	}
      else if (TARGET_BMI && GET_CODE (XEXP (x,1)) == NOT)
	{
	  *total = cost->add
		   + rtx_cost (XEXP (x, 0), mode, outer_code, opno, speed)
		   + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
			       outer_code, opno, speed);
	  return true;
	}
      else
	*total = cost->add;
      return false;

    case NOT:
      if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
	{
	  /* (not (xor ...)) can be a single insn in AVX512.  */
	  if (GET_CODE (XEXP (x, 0)) == XOR && TARGET_AVX512F
	      && (GET_MODE_SIZE (mode) == 64
		  || (TARGET_AVX512VL
		      && (GET_MODE_SIZE (mode) == 32
			  || GET_MODE_SIZE (mode) == 16))))
	    {
	      *total = ix86_vec_cost (mode, cost->sse_op)
		       + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
				   outer_code, opno, speed)
		       + rtx_cost (XEXP (XEXP (x, 0), 1), mode,
				   outer_code, opno, speed);
	      return true;
	    }

	  // vnot is pxor -1.
	  *total = ix86_vec_cost (mode, cost->sse_op) + 1;
	}
      else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	*total = cost->add * 2;
      else
	*total = cost->add;
      return false;

    case NEG:
      if (SSE_FLOAT_MODE_SSEMATH_OR_HF_P (mode))
	*total = cost->sse_op;
      else if (X87_FLOAT_MODE_P (mode))
	*total = cost->fchs;
      else if (FLOAT_MODE_P (mode))
	*total = ix86_vec_cost (mode, cost->sse_op);
      else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
	*total = ix86_vec_cost (mode, cost->sse_op);
      else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	*total = cost->add * 3;
      else
	*total = cost->add;
      return false;

    case COMPARE:
      rtx op0, op1;
      op0 = XEXP (x, 0);
      op1 = XEXP (x, 1);
      if (GET_CODE (op0) == ZERO_EXTRACT
	  && XEXP (op0, 1) == const1_rtx
	  && CONST_INT_P (XEXP (op0, 2))
	  && op1 == const0_rtx)
	{
	  /* This kind of construct is implemented using test[bwl].
	     Treat it as if we had an AND.  */
	  mode = GET_MODE (XEXP (op0, 0));
	  *total = (cost->add
		    + rtx_cost (XEXP (op0, 0), mode, outer_code,
				opno, speed)
		    + rtx_cost (const1_rtx, mode, outer_code, opno, speed));
	  return true;
	}

      if (GET_CODE (op0) == PLUS && rtx_equal_p (XEXP (op0, 0), op1))
	{
	  /* This is an overflow detection, count it as a normal compare.  */
	  *total = rtx_cost (op0, GET_MODE (op0), COMPARE, 0, speed);
	  return true;
	}

      rtx geu;
      /* Match x
	 (compare:CCC (neg:QI (geu:QI (reg:CC_CCC FLAGS_REG) (const_int 0)))
		      (ltu:QI (reg:CC_CCC FLAGS_REG) (const_int 0)))  */
      if (mode == CCCmode
	  && GET_CODE (op0) == NEG
	  && GET_CODE (geu = XEXP (op0, 0)) == GEU
	  && REG_P (XEXP (geu, 0))
	  && (GET_MODE (XEXP (geu, 0)) == CCCmode
	      || GET_MODE (XEXP (geu, 0)) == CCmode)
	  && REGNO (XEXP (geu, 0)) == FLAGS_REG
	  && XEXP (geu, 1) == const0_rtx
	  && GET_CODE (op1) == LTU
	  && REG_P (XEXP (op1, 0))
	  && GET_MODE (XEXP (op1, 0)) == GET_MODE (XEXP (geu, 0))
	  && REGNO (XEXP (op1, 0)) == FLAGS_REG
	  && XEXP (op1, 1) == const0_rtx)
	{
	  /* This is *setcc_qi_addqi3_cconly_overflow_1_* patterns, a nop.  */
	  *total = 0;
	  return true;
	}
      /* Match x
	 (compare:CCC (neg:QI (ltu:QI (reg:CCC FLAGS_REG) (const_int 0)))
		      (geu:QI (reg:CCC FLAGS_REG) (const_int 0)))  */
      if (mode == CCCmode
	  && GET_CODE (op0) == NEG
	  && GET_CODE (XEXP (op0, 0)) == LTU
	  && REG_P (XEXP (XEXP (op0, 0), 0))
	  && GET_MODE (XEXP (XEXP (op0, 0), 0)) == CCCmode
	  && REGNO (XEXP (XEXP (op0, 0), 0)) == FLAGS_REG
	  && XEXP (XEXP (op0, 0), 1) == const0_rtx
	  && GET_CODE (op1) == GEU
	  && REG_P (XEXP (op1, 0))
	  && GET_MODE (XEXP (op1, 0)) == CCCmode
	  && REGNO (XEXP (op1, 0)) == FLAGS_REG
	  && XEXP (op1, 1) == const0_rtx)
	{
	  /* This is *x86_cmc.  */
	  if (!speed)
	    *total = COSTS_N_BYTES (1);
	  else if (TARGET_SLOW_STC)
	    *total = COSTS_N_INSNS (2);
	  else 
	    *total = COSTS_N_INSNS (1);
	  return true;
	}

      if (SCALAR_INT_MODE_P (GET_MODE (op0))
	  && GET_MODE_SIZE (GET_MODE (op0)) > UNITS_PER_WORD)
	{
	  if (op1 == const0_rtx)
	    *total = cost->add
		     + rtx_cost (op0, GET_MODE (op0), outer_code, opno, speed);
	  else
	    *total = 3*cost->add