aboutsummaryrefslogtreecommitdiff
path: root/gcc/config/rs6000/beos.h
diff options
context:
space:
mode:
Diffstat (limited to 'gcc/config/rs6000/beos.h')
-rw-r--r--gcc/config/rs6000/beos.h3
1 files changed, 2 insertions, 1 deletions
diff --git a/gcc/config/rs6000/beos.h b/gcc/config/rs6000/beos.h
index a9e88ac..b7a05e4 100644
--- a/gcc/config/rs6000/beos.h
+++ b/gcc/config/rs6000/beos.h
@@ -1,5 +1,6 @@
/* Definitions of target machine for GNU compiler, for BeOS.
- Copyright (C) 1997, 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
+ Copyright (C) 1997, 2000, 2001, 2002, 2003, 2005
+ Free Software Foundation, Inc.
Contributed by Fred Fish (fnf@cygnus.com), based on aix41.h
from David Edelsohn (edelsohn@npac.syr.edu).
generated by cgit v1.1 at 2025-09-27 06:30:20 +0000
href='#n106'>106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989 7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181 8182 8183 8184 8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201 8202 8203 8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253 8254 8255 8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340 8341 8342 8343 8344 8345 8346 8347 8348 8349 8350 8351 8352 8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379 8380 8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426 8427 8428 8429 8430 8431 8432 8433 8434 8435 8436 8437 8438 8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474 8475 8476 8477 8478 8479 8480 8481 8482 8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494 8495 8496 8497 8498 8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619 8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714 8715 8716 8717 8718 8719 8720 8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802 8803 8804 8805 8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816 8817 8818 8819 8820 8821 8822 8823 8824 8825 8826 8827 8828 8829 8830 8831 8832 8833 8834 8835 8836 8837 8838 8839 8840 8841 8842 8843 8844 8845 8846 8847 8848 8849 8850 8851 8852 8853 8854 8855 8856 8857 8858 8859 8860 8861 8862 8863 8864 8865 8866 8867 8868 8869 8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883 8884 8885 8886 8887 8888 8889 8890 8891 8892 8893 8894 8895 8896 8897 8898 8899 8900 8901 8902 8903 8904 8905 8906 8907 8908 8909 8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930 8931 8932 8933 8934 8935 8936 8937 8938 8939 8940 8941 8942 8943 8944 8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956 8957 8958 8959 8960 8961 8962 8963 8964 8965 8966 8967 8968 8969 8970 8971 8972 8973 8974 8975 8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990 8991 8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002 9003 9004 9005 9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022 9023 9024 9025 9026 9027 9028 9029 9030 9031 9032 9033 9034 9035 9036 9037 9038 9039 9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051 9052 9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063 9064 9065 9066 9067 9068 9069 9070 9071 9072 9073 9074 9075 9076 9077 9078 9079 9080 9081 9082 9083 9084 9085 9086 9087 9088 9089 9090 9091 9092 9093 9094 9095 9096 9097 9098 9099 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109 9110 9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151 9152 9153 9154 9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172 9173 9174 9175 9176 9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201 9202 9203 9204 9205 9206 9207 9208 9209 9210 9211 9212 9213 9214 9215 9216 9217 9218 9219 9220 9221 9222 9223 9224 9225 9226 9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254 9255 9256 9257 9258 9259 9260 9261 9262 9263 9264 9265 9266 9267 9268 9269 9270 9271 9272 9273 9274 9275 9276 9277 9278 9279 9280 9281 9282 9283 9284 9285 9286 9287 9288 9289 9290 9291 9292 9293 9294 9295 9296 9297 9298 9299 9300 9301 9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317 9318 9319 9320 9321 9322 9323 9324 9325 9326 9327 9328 9329 9330 9331 9332 9333 9334 9335 9336 9337 9338 9339 9340 9341 9342 9343 9344 9345 9346 9347 9348 9349 9350 9351 9352 9353 9354 9355 9356 9357 9358 9359 9360 9361 9362 9363 9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412 9413 9414 9415 9416 9417 9418 9419 9420 9421 9422 9423 9424 9425 9426 9427 9428 9429 9430 9431 9432 9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449 9450 9451 9452 9453 9454 9455 9456 9457 9458 9459 9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530 9531 9532 9533 9534 9535 9536 9537 9538 9539 9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583 9584 9585 9586 9587 9588 9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608 9609 9610 9611 9612 9613 9614 9615 9616 9617 9618 9619 9620 9621 9622 9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634 9635 9636 9637 9638 9639 9640 9641 9642 9643 9644 9645 9646 9647 9648 9649 9650 9651 9652 9653 9654 9655 9656 9657 9658 9659 9660 9661 9662 9663 9664 9665 9666 9667 9668 9669 9670 9671 9672 9673 9674 9675 9676 9677 9678 9679 9680 9681 9682 9683 9684 9685 9686 9687 9688 9689 9690 9691 9692 9693 9694 9695 9696 9697 9698 9699 9700 9701 9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719 9720 9721 9722 9723 9724 9725 9726 9727 9728 9729 9730 9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746 9747 9748 9749 9750 9751 9752 9753 9754 9755 9756 9757 9758 9759 9760 9761 9762 9763 9764 9765 9766 9767 9768 9769 9770 9771 9772 9773 9774 9775 9776 9777 9778 9779 9780 9781 9782 9783 9784 9785 9786 9787 9788 9789 9790 9791 9792 9793 9794 9795 9796 9797 9798 9799 9800 9801 9802 9803 9804 9805 9806 9807 9808 9809 9810 9811 9812 9813 9814 9815 9816 9817 9818 9819 9820 9821 9822 9823 9824 9825 9826 9827 9828 9829 9830 9831 9832 9833 9834 9835 9836 9837 9838 9839 9840 9841 9842 9843 9844 9845 9846 9847 9848 9849 9850 9851 9852 9853 9854 9855 9856 9857 9858 9859 9860 9861 9862 9863 9864 9865 9866 9867 9868 9869 9870 9871 9872 9873 9874 9875 9876 9877 9878 9879 9880 9881 9882 9883 9884 9885 9886 9887 9888 9889 9890 9891 9892 9893 9894 9895 9896 9897 9898 9899 9900 9901 9902 9903 9904 9905 9906 9907 9908 9909 9910 9911 9912 9913 9914 9915 9916 9917 9918 9919 9920 9921 9922 9923 9924 9925 9926 9927 9928 9929 9930 9931 9932 9933 9934 9935 9936 9937 9938 9939 9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962 9963 9964 9965 9966 9967 9968 9969 9970 9971 9972 9973 9974 9975 9976 9977 9978 9979 9980 9981 9982 9983 9984 9985 9986 9987 9988 9989 9990 9991 9992 9993 9994 9995 9996 9997 9998 9999 10000 10001 10002 10003 10004 10005 10006 10007 10008 10009 10010 10011 10012 10013 10014 10015 10016 10017 10018 10019 10020 10021 10022 10023 10024 10025 10026 10027 10028 10029 10030 10031 10032 10033 10034 10035 10036 10037 10038 10039 10040 10041 10042 10043 10044 10045 10046 10047 10048 10049 10050 10051 10052 10053 10054 10055 10056 10057 10058 10059 10060 10061 10062 10063 10064 10065 10066 10067 10068 10069 10070 10071 10072 10073 10074 10075 10076 10077 10078 10079 10080 10081 10082 10083 10084 10085 10086 10087 10088 10089 10090 10091 10092 10093 10094 10095 10096 10097 10098 10099 10100 10101 10102 10103 10104 10105 10106 10107 10108 10109 10110 10111 10112 10113 10114 10115 10116 10117 10118 10119 10120 10121 10122 10123 10124 10125 10126 10127 10128 10129 10130 10131 10132 10133 10134 10135 10136 10137 10138 10139 10140 10141 10142 10143 10144 10145 10146 10147 10148 10149 10150 10151 10152 10153 10154 10155 10156 10157 10158 10159 10160 10161 10162 10163 10164 10165 10166 10167 10168 10169 10170 10171 10172 10173 10174 10175 10176 10177 10178 10179 10180 10181 10182 10183 10184 10185 10186 10187 10188 10189 10190 10191 10192 10193 10194 10195 10196 10197 10198 10199 10200 10201 10202 10203 10204 10205 10206 10207 10208 10209 10210 10211 10212 10213 10214 10215 10216 10217 10218 10219 10220 10221 10222 10223 10224 10225 10226 10227 10228 10229 10230 10231 10232 10233 10234 10235 10236 10237 10238 10239 10240 10241 10242 10243 10244 10245 10246 10247 10248 10249 10250 10251 10252 10253 10254 10255 10256 10257 10258 10259 10260 10261 10262 10263 10264 10265 10266 10267 10268 10269 10270 10271 10272 10273 10274 10275 10276 10277 10278 10279 10280 10281 10282 10283 10284 10285 10286 10287 10288 10289 10290 10291 10292 10293 10294 10295 10296 10297 10298 10299 10300 10301 10302 10303 10304 10305 10306 10307 10308 10309 10310 10311 10312 10313 10314 10315 10316 10317 10318 10319 10320 10321 10322 10323 10324 10325 10326 10327 10328 10329 10330 10331 10332 10333 10334 10335 10336 10337 10338 10339 10340 10341 10342 10343 10344 10345 10346 10347 10348 10349 10350 10351 10352 10353 10354 10355 10356 10357 10358 10359 10360 10361 10362 10363 10364 10365 10366 10367 10368 10369 10370 10371 10372 10373 10374 10375 10376 10377 10378 10379 10380 10381 10382 10383 10384 10385 10386 10387 10388 10389 10390 10391 10392 10393 10394 10395 10396 10397 10398 10399 10400 10401 10402 10403 10404 10405 10406 10407 10408 10409 10410 10411 10412 10413 10414 10415 10416 10417 10418 10419 10420 10421 10422 10423 10424 10425 10426 10427 10428 10429 10430 10431 10432 10433 10434 10435 10436 10437 10438 10439 10440 10441 10442 10443 10444 10445 10446 10447 10448 10449 10450 10451 10452 10453 10454 10455 10456 10457 10458 10459 10460 10461 10462 10463 10464 10465 10466 10467 10468 10469 10470 10471 10472 10473 10474 10475 10476 10477 10478 10479 10480 10481 10482 10483 10484 10485 10486 10487 10488 10489 10490 10491 10492 10493 10494 10495 10496 10497 10498 10499 10500 10501 10502 10503 10504 10505 10506 10507 10508 10509 10510 10511 10512 10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523 10524 10525 10526 10527 10528 10529 10530 10531 10532 10533 10534 10535 10536 10537 10538 10539 10540 10541 10542 10543 10544 10545 10546 10547 10548 10549 10550 10551 10552 10553 10554 10555 10556 10557 10558 10559 10560 10561 10562 10563 10564 10565 10566 10567 10568 10569 10570 10571 10572 10573 10574 10575 10576 10577 10578 10579 10580 10581 10582 10583 10584 10585 10586 10587 10588 10589 10590 10591 10592 10593 10594 10595 10596 10597 10598 10599 10600 10601 10602 10603 10604 10605 10606 10607 10608 10609 10610 10611 10612 10613 10614 10615 10616 10617 10618 10619 10620 10621 10622 10623 10624 10625 10626 10627 10628 10629 10630 10631 10632 10633 10634 10635 10636 10637 10638 10639 10640 10641 10642 10643 10644 10645 10646 10647 10648 10649 10650 10651 10652 10653 10654 10655 10656 10657 10658 10659 10660 10661 10662 10663 10664 10665 10666 10667 10668 10669 10670 10671 10672 10673 10674 10675 10676 10677 10678 10679 10680 10681 10682 10683 10684 10685 10686 10687 10688 10689 10690 10691 10692 10693 10694 10695 10696 10697 10698 10699 10700 10701 10702 10703 10704 10705 10706 10707 10708 10709 10710 10711 10712 10713 10714 10715 10716 10717 10718 10719 10720 10721 10722 10723 10724 10725 10726 10727 10728 10729 10730 10731 10732 10733 10734 10735 10736 10737 10738 10739 10740 10741 10742 10743 10744 10745 10746 10747 10748 10749 10750 10751 10752 10753 10754 10755 10756 10757 10758 10759 10760 10761 10762 10763 10764 10765 10766 10767 10768 10769 10770 10771 10772 10773 10774 10775 10776 10777 10778 10779 10780 10781 10782 10783 10784 10785 10786 10787 10788 10789 10790 10791 10792 10793 10794 10795 10796 10797 10798 10799 10800 10801 10802 10803 10804 10805 10806 10807 10808 10809 10810 10811 10812 10813 10814 10815 10816 10817 10818 10819 10820 10821 10822 10823 10824 10825 10826 10827 10828 10829 10830 10831 10832 10833 10834 10835 10836 10837 10838 10839 10840 10841 10842 10843 10844 10845 10846 10847 10848 10849 10850 10851 10852 10853 10854 10855 10856 10857 10858 10859 10860 10861 10862 10863 10864 10865 10866 10867 10868 10869 10870 10871 10872 10873 10874 10875 10876 10877 10878 10879 10880 10881 10882 10883 10884 10885 10886 10887 10888 10889 10890 10891 10892 10893 10894 10895 10896 10897 10898 10899 10900 10901 10902 10903 10904 10905 10906 10907 10908 10909 10910 10911 10912 10913 10914 10915 10916 10917 10918 10919 10920 10921 10922 10923 10924 10925 10926 10927 10928 10929 10930 10931 10932 10933 10934 10935 10936 10937 10938 10939 10940 10941 10942 10943 10944 10945 10946 10947 10948 10949 10950 10951 10952 10953 10954 10955 10956 10957 10958 10959 10960 10961 10962 10963 10964 10965 10966 10967 10968 10969 10970 10971 10972 10973 10974 10975 10976 10977 10978 10979 10980 10981 10982 10983 10984 10985 10986 10987 10988 10989 10990 10991 10992 10993 10994 10995 10996 10997 10998 10999 11000 11001 11002 11003 11004 11005 11006 11007 11008 11009 11010 11011 11012 11013 11014 11015 11016 11017 11018 11019 11020 11021 11022 11023 11024 11025 11026 11027 11028 11029 11030 11031 11032 11033 11034 11035 11036 11037 11038 11039 11040 11041 11042 11043 11044 11045 11046 11047 11048 11049 11050 11051 11052 11053 11054 11055 11056 11057 11058 11059 11060 11061 11062 11063 11064 11065 11066 11067 11068 11069 11070 11071 11072 11073 11074 11075 11076 11077 11078 11079 11080 11081 11082 11083 11084 11085 11086 11087 11088 11089 11090 11091 11092 11093 11094 11095 11096 11097 11098 11099 11100 11101 11102 11103 11104 11105 11106 11107 11108 11109 11110 11111 11112 11113 11114 11115 11116 11117 11118 11119 11120 11121 11122 11123 11124 11125 11126 11127 11128 11129 11130 11131 11132 11133 11134 11135 11136 11137 11138 11139 11140 11141 11142 11143 11144 11145 11146 11147 11148 11149 11150 11151 11152 11153 11154 11155 11156 11157 11158 11159 11160 11161 11162 11163 11164 11165 11166 11167 11168 11169 11170 11171 11172 11173 11174 11175 11176 11177 11178 11179 11180 11181 11182 11183 11184 11185 11186 11187 11188 11189 11190 11191 11192 11193 11194 11195 11196 11197 11198 11199 11200 11201 11202 11203 11204 11205 11206 11207 11208 11209 11210 11211 11212 11213 11214 11215 11216 11217 11218 11219 11220 11221 11222 11223 11224 11225 11226 11227 11228 11229 11230 11231 11232 11233 11234 11235 11236 11237 11238 11239 11240 11241 11242 11243 11244 11245 11246 11247 11248 11249 11250 11251 11252 11253 11254 11255 11256 11257 11258 11259 11260 11261 11262 11263 11264 11265 11266 11267 11268 11269 11270 11271 11272 11273 11274 11275 11276 11277 11278 11279 11280 11281 11282 11283 11284 11285 11286 11287 11288 11289 11290 11291 11292 11293 11294 11295 11296 11297 11298 11299 11300 11301 11302 11303 11304 11305 11306 11307 11308 11309 11310 11311 11312 11313 11314 11315 11316 11317 11318 11319 11320 11321 11322 11323 11324 11325 11326 11327 11328 11329 11330 11331 11332 11333 11334 11335 11336 11337 11338 11339 11340 11341 11342 11343 11344 11345 11346 11347 11348 11349 11350 11351 11352 11353 11354 11355 11356 11357 11358 11359 11360 11361 11362 11363 11364 11365 11366 11367 11368 11369 11370 11371 11372 11373 11374 11375 11376 11377 11378 11379 11380 11381 11382 11383 11384 11385 11386 11387 11388 11389 11390 11391 11392 11393 11394 11395 11396 11397 11398 11399 11400 11401 11402 11403 11404 11405 11406 11407 11408 11409 11410 11411 11412 11413 11414 11415 11416 11417 11418 11419 11420 11421 11422 11423 11424 11425 11426 11427 11428 11429 11430 11431 11432 11433 11434 11435 11436 11437 11438 11439 11440 11441 11442 11443 11444 11445 11446 11447 11448 11449 11450 11451 11452 11453 11454 11455 11456 11457 11458 11459 11460 11461 11462 11463 11464 11465 11466 11467 11468 11469 11470 11471 11472 11473 11474 11475 11476 11477 11478 11479 11480 11481 11482 11483 11484 11485 11486 11487 11488 11489 11490 11491 11492 11493 11494 11495 11496 11497 11498 11499 11500 11501 11502 11503 11504 11505 11506 11507 11508 11509 11510 11511 11512 11513 11514 11515 11516 11517 11518 11519 11520 11521 11522 11523 11524 11525 11526 11527 11528 11529 11530 11531 11532 11533 11534 11535 11536 11537 11538 11539 11540 11541 11542 11543 11544 11545 11546 11547 11548 11549 11550 11551 11552 11553 11554 11555 11556 11557 11558 11559 11560 11561 11562 11563 11564 11565 11566 11567 11568 11569 11570 11571 11572 11573 11574 11575 11576 11577 11578 11579 11580 11581 11582 11583 11584 11585 11586 11587 11588 11589 11590 11591 11592 11593 11594 11595 11596 11597 11598 11599 11600 11601 11602 11603 11604 11605 11606 11607 11608 11609 11610 11611 11612 11613 11614 11615 11616 11617 11618 11619 11620 11621 11622 11623 11624 11625 11626 11627 11628 11629 11630 11631 11632 11633 11634 11635 11636 11637 11638 11639 11640 11641 11642 11643 11644 11645 11646 11647 11648 11649 11650 11651 11652 11653 11654 11655 11656 11657 11658 11659 11660 11661 11662 11663 11664 11665 11666 11667 11668 11669 11670 11671 11672 11673 11674 11675 11676 11677 11678 11679 11680 11681 11682 11683 11684 11685 11686 11687 11688 11689 11690 11691 11692 11693 11694 11695 11696 11697 11698 11699 11700 11701 11702 11703 11704 11705 11706 11707 11708 11709 11710 11711 11712 11713 11714 11715 11716 11717 11718 11719 11720 11721 11722 11723 11724 11725 11726 11727 11728 11729 11730 11731 11732 11733 11734 11735 11736 11737 11738 11739 11740 11741 11742 11743 11744 11745 11746 11747 11748 11749 11750 11751 11752 11753 11754 11755 11756 11757 11758 11759 11760 11761 11762 11763 11764 11765 11766 11767 11768 11769 11770 11771 11772 11773 11774 11775 11776 11777 11778 11779 11780 11781 11782 11783 11784 11785 11786 11787 11788 11789 11790 11791 11792 11793 11794 11795 11796 11797 11798 11799 11800 11801 11802 11803 11804 11805 11806 11807 11808 11809 11810 11811 11812 11813 11814 11815 11816 11817 11818 11819 11820 11821 11822 11823 11824 11825 11826 11827 11828 11829 11830 11831 11832 11833 11834 11835 11836 11837 11838 11839 11840 11841 11842 11843 11844 11845 11846 11847 11848 11849 11850 11851 11852 11853 11854 11855 11856 11857 11858 11859 11860 11861 11862 11863 11864 11865 11866 11867 11868 11869 11870 11871 11872 11873 11874 11875 11876 11877 11878 11879 11880 11881 11882 11883 11884 11885 11886 11887 11888 11889 11890 11891 11892 11893 11894 11895 11896 11897 11898 11899 11900 11901 11902 11903 11904 11905 11906 11907 11908 11909 11910 11911 11912 11913 11914 11915 11916 11917 11918 11919 11920 11921 11922 11923 11924 11925 11926 11927 11928 11929 11930 11931 11932 11933 11934 11935 11936 11937 11938 11939 11940 11941 11942 11943 11944 11945 11946 11947 11948 11949 11950 11951 11952 11953 11954 11955 11956 11957 11958 11959 11960 11961 11962 11963 11964 11965 11966 11967 11968 11969 11970 11971 11972 11973 11974 11975 11976 11977 11978 11979 11980 11981 11982 11983 11984 11985 11986 11987 11988 11989 11990 11991 11992 11993 11994 11995 11996 11997 11998 11999 12000 12001 12002 12003 12004 12005 12006 12007 12008 12009 12010 12011 12012 12013 12014 12015 12016 12017 12018 12019 12020 12021 12022 12023 12024 12025 12026 12027 12028 12029 12030 12031 12032 12033 12034 12035 12036 12037 12038 12039 12040 12041 12042 12043 12044 12045 12046 12047 12048 12049 12050 12051 12052 12053 12054 12055 12056 12057 12058 12059 12060 12061 12062 12063 12064 12065 12066 12067 12068 12069 12070 12071 12072 12073 12074 12075 12076 12077 12078 12079 12080 12081 12082 12083 12084 12085 12086 12087 12088 12089 12090 12091 12092 12093 12094 12095 12096 12097 12098 12099 12100 12101 12102 12103 12104 12105 12106 12107 12108 12109 12110 12111 12112 12113 12114 12115 12116 12117 12118 12119 12120 12121 12122 12123 12124 12125 12126 12127 12128 12129 12130 12131 12132 12133 12134 12135 12136 12137 12138 12139 12140 12141 12142 12143 12144 12145 12146 12147 12148 12149 12150 12151 12152 12153 12154 12155 12156 12157 12158 12159 12160 12161 12162 12163 12164 12165 12166 12167 12168 12169 12170 12171 12172 12173 12174 12175 12176 12177 12178 12179 12180 12181 12182 12183 12184 12185 12186 12187 12188 12189 12190 12191 12192 12193 12194 12195 12196 12197 12198 12199 12200 12201 12202 12203 12204 12205 12206 12207 12208 12209 12210 12211 12212 12213 12214 12215 12216 12217 12218 12219 12220 12221 12222 12223 12224 12225 12226 12227 12228 12229 12230 12231 12232 12233 12234 12235 12236 12237 12238 12239 12240 12241 12242 12243 12244 12245 12246 12247 12248 12249 12250 12251 12252 12253 12254 12255 12256 12257 12258 12259 12260 12261 12262 12263 12264 12265 12266 12267 12268 12269 12270 12271 12272 12273 12274 12275 12276 12277 12278 12279 12280 12281 12282 12283 12284 12285 12286 12287 12288 12289 12290 12291 12292 12293 12294 12295 12296 12297 12298 12299 12300 12301 12302 12303 12304 12305 12306 12307 12308 12309 12310 12311 12312 12313 12314 12315 12316 12317 12318 12319 12320 12321 12322 12323 12324 12325 12326 12327 12328 12329 12330 12331 12332 12333 12334 12335 12336 12337 12338 12339 12340 12341 12342 12343 12344 12345 12346 12347 12348 12349 12350 12351 12352 12353 12354 12355 12356 12357 12358 12359 12360 12361 12362 12363 12364 12365 12366 12367 12368 12369 12370 12371 12372 12373 12374 12375 12376 12377 12378 12379 12380 12381 12382 12383 12384 12385 12386 12387 12388 12389 12390 12391 12392 12393 12394 12395 12396 12397 12398 12399 12400 12401 12402 12403 12404 12405 12406 12407 12408 12409 12410 12411 12412 12413 12414 12415 12416 12417 12418 12419 12420 12421 12422 12423 12424 12425 12426 12427 12428 12429 12430 12431 12432 12433 12434 12435 12436 12437 12438 12439 12440 12441 12442 12443 12444 12445 12446 12447 12448 12449 12450 12451 12452 12453 12454 12455 12456 12457 12458 12459 12460 12461 12462 12463 12464 12465 12466 12467 12468 12469 12470 12471 12472 12473 12474 12475 12476 12477 12478 12479 12480 12481 12482 12483 12484 12485 12486 12487 12488 12489 12490 12491 12492 12493 12494 12495 12496 12497 12498 12499 12500 12501 12502 12503 12504 12505 12506 12507 12508 12509 12510 12511 12512 12513 12514 12515 12516 12517 12518 12519 12520 12521 12522 12523 12524 12525 12526 12527 12528 12529 12530 12531 12532 12533 12534 12535 12536 12537 12538 12539 12540 12541 12542 12543 12544 12545 12546 12547 12548 12549 12550 12551 12552 12553 12554 12555 12556 12557 12558 12559 12560 12561 12562 12563 12564 12565 12566 12567 12568 12569 12570 12571 12572 12573 12574 12575 12576 12577 12578 12579 12580 12581 12582 12583 12584 12585 12586 12587 12588 12589 12590 12591 12592 12593 12594 12595 12596 12597 12598 12599 12600 12601 12602 12603 12604 12605 12606 12607 12608 12609 12610 12611 12612 12613 12614 12615 12616 12617 12618 12619 12620 12621 12622 12623 12624 12625 12626 12627 12628 12629 12630 12631 12632 12633 12634 12635 12636 12637 12638 12639 12640 12641 12642 12643 12644 12645 12646 12647 12648 12649 12650 12651 12652 12653 12654 12655 12656 12657 12658 12659 12660 12661 12662 12663 12664 12665 12666 12667 12668 12669 12670 12671 12672 12673 12674 12675 12676 12677 12678 12679 12680 12681 12682 12683 12684 12685 12686 12687 12688 12689 12690 12691 12692 12693 12694 12695 12696 12697 12698 12699 12700 12701 12702 12703 12704 12705 12706 12707 12708 12709 12710 12711 12712 12713 12714 12715 12716 12717 12718 12719 12720 12721 12722 12723 12724 12725 12726 12727 12728 12729 12730 12731 12732 12733 12734 12735 12736 12737 12738 12739 12740 12741 12742 12743 12744 12745 12746 12747 12748 12749 12750 12751 12752 12753 12754 12755 12756 12757 12758 12759 12760 12761 12762 12763 12764 12765 12766 12767 12768 12769 12770 12771 12772 12773 12774 12775 12776 12777 12778 12779 12780 12781 12782 12783 12784 12785 12786 12787 12788 12789 12790 12791 12792 12793 12794 12795 12796 12797 12798 12799 12800 12801 12802 12803 12804 12805 12806 12807 12808 12809 12810 12811 12812 12813 12814 12815 12816 12817 12818 12819 12820 12821 12822 12823 12824 12825 12826 12827 12828 12829 12830 12831 12832 12833 12834 12835 12836 12837 12838 12839 12840 12841 12842 12843 12844 12845 12846 12847 12848 12849 12850 12851 12852 12853 12854 12855 12856 12857 12858 12859 12860 12861 12862 12863 12864 12865 12866 12867 12868 12869 12870 12871 12872 12873 12874 12875 12876 12877 12878 12879 12880 12881 12882 12883 12884 12885 12886 12887 12888 12889 12890 12891 12892 12893 12894 12895 12896 12897 12898 12899 12900 12901 12902 12903 12904 12905 12906 12907 12908 12909 12910 12911 12912 12913 12914 12915 12916 12917 12918 12919 12920 12921 12922 12923 12924 12925 12926 12927 12928 12929 12930 12931 12932 12933 12934 12935 12936 12937 12938 12939 12940 12941 12942 12943 12944 12945 12946 12947 12948 12949 12950 12951 12952 12953 12954 12955 12956 12957 12958 12959 12960 12961 12962 12963 12964 12965 12966 12967 12968 12969 12970 12971 12972 12973 12974 12975 12976 12977 12978 12979 12980 12981 12982 12983 12984 12985 12986 12987 12988 12989 12990 12991 12992 12993 12994 12995 12996 12997 12998 12999 13000 13001 13002 13003 13004 13005 13006 13007 13008 13009 13010 13011 13012 13013 13014 13015 13016 13017 13018 13019 13020 13021 13022 13023 13024 13025 13026 13027 13028 13029 13030 13031 13032 13033 13034 13035 13036 13037 13038 13039 13040 13041 13042 13043 13044 13045 13046 13047 13048 13049 13050 13051 13052 13053 13054 13055 13056 13057 13058 13059 13060 13061 13062 13063 13064 13065 13066 13067 13068 13069 13070 13071 13072 13073 13074 13075 13076 13077 13078 13079 13080 13081 13082 13083 13084 13085 13086 13087 13088 13089 13090 13091 13092 13093 13094 13095 13096 13097 13098 13099 13100 13101 13102 13103 13104 13105 13106 13107 13108 13109 13110 13111 13112 13113 13114 13115 13116 13117 13118 13119 13120 13121 13122 13123 13124 13125 13126 13127 13128 13129 13130 13131 13132 13133 13134 13135 13136 13137 13138 13139 13140 13141 13142 13143 13144 13145 13146 13147 13148 13149 13150 13151 13152 13153 13154 13155 13156 13157 13158 13159 13160 13161 13162 13163 13164 13165 13166 13167 13168 13169 13170 13171 13172 13173 13174 13175 13176 13177 13178 13179 13180 13181 13182 13183 13184 13185 13186 13187 13188 13189 13190 13191 13192 13193 13194 13195 13196 13197 13198 13199 13200 13201 13202 13203 13204 13205 13206 13207 13208 13209 13210 13211 13212 13213 13214 13215 13216 13217 13218 13219 13220 13221 13222 13223 13224 13225 13226 13227 13228 13229 13230 13231 13232 13233 13234 13235 13236 13237 13238 13239 13240 13241 13242 13243 13244 13245 13246 13247 13248 13249 13250 13251 13252 13253 13254 13255 13256 13257 13258 13259 13260 13261 13262 13263 13264 13265 13266 13267 13268 13269 13270 13271 13272 13273 13274 13275 13276 13277 13278 13279 13280 13281 13282 13283 13284 13285 13286 13287 13288 13289 13290 13291 13292 13293 13294 13295 13296 13297 13298 13299 13300 13301 13302 13303 13304 13305 13306 13307 13308 13309 13310 13311 13312 13313 13314 13315 13316 13317 13318 13319 13320 13321 13322 13323 13324 13325 13326 13327 13328 13329 13330 13331 13332 13333 13334 13335 13336 13337 13338 13339 13340 13341 13342 13343 13344 13345 13346 13347 13348 13349 13350 13351 13352 13353 13354 13355 13356 13357 13358 13359 13360 13361 13362 13363 13364 13365 13366 13367 13368 13369 13370 13371 13372 13373 13374 13375 13376 13377 13378 13379 13380 13381 13382 13383 13384 13385 13386 13387 13388 13389 13390 13391 13392 13393 13394 13395 13396 13397 13398 13399 13400 13401 13402 13403 13404 13405 13406 13407 13408 13409 13410 13411 13412 13413 13414 13415 13416 13417 13418 13419 13420 13421 13422 13423 13424 13425 13426 13427 13428 13429 13430 13431 13432 13433 13434 13435 13436 13437 13438 13439 13440 13441 13442 13443 13444 13445 13446 13447 13448 13449 13450 13451 13452 13453 13454 13455 13456 13457 13458 13459 13460 13461 13462 13463 13464 13465 13466 13467 13468 13469 13470 13471 13472 13473 13474 13475 13476 13477 13478 13479 13480 13481 13482 13483 13484 13485 13486 13487 13488 13489 13490 13491 13492 13493 13494 13495 13496 13497 13498 13499 13500 13501 13502 13503 13504 13505 13506 13507 13508 13509 13510 13511 13512 13513 13514 13515 13516 13517 13518 13519 13520 13521 13522 13523 13524 13525 13526 13527 13528 13529 13530 13531 13532 13533 13534 13535 13536 13537 13538 13539 13540 13541 13542 13543 13544 13545 13546 13547 13548 13549 13550 13551 13552 13553 13554 13555 13556 13557 13558 13559 13560 13561 13562 13563 13564 13565 13566 13567 13568 13569 13570 13571 13572 13573 13574 13575 13576 13577 13578 13579 13580 13581 13582 13583 13584 13585 13586 13587 13588 13589 13590 13591 13592 13593 13594 13595 13596 13597 13598 13599 13600 13601 13602 13603 13604 13605 13606 13607 13608 13609 13610 13611 13612 13613 13614 13615 13616 13617 13618 13619 13620 13621 13622 13623 13624 13625 13626 13627 13628 13629 13630 13631 13632 13633 13634 13635 13636 13637 13638 13639 13640 13641 13642 13643 13644 13645 13646 13647 13648 13649 13650 13651 13652 13653 13654 13655 13656 13657 13658 13659 13660 13661 13662 13663 13664 13665 13666 13667 13668 13669 13670 13671 13672 13673 13674 13675 13676 13677 13678 13679 13680 13681 13682 13683 13684 13685 13686 13687 13688 13689 13690 13691 13692 13693 13694 13695 13696 13697 13698 13699 13700 13701 13702 13703 13704 13705 13706 13707 13708 13709 13710 13711 13712 13713 13714 13715 13716 13717 13718 13719 13720 13721 13722 13723 13724 13725 13726 13727 13728 13729 13730 13731 13732 13733 13734 13735 13736 13737 13738 13739 13740 13741 13742 13743 13744 13745 13746 13747 13748 13749 13750 13751 13752 13753 13754 13755 13756 13757 13758 13759 13760 13761 13762 13763 13764 13765 13766 13767 13768 13769 13770 13771 13772 13773 13774 13775 13776 13777 13778 13779 13780 13781 13782 13783 13784 13785 13786 13787 13788 13789 13790 13791 13792 13793 13794 13795 13796 13797 13798 13799 13800 13801 13802 13803 13804 13805 13806 13807 13808 13809 13810 13811 13812 13813 13814 13815 13816 13817 13818 13819 13820 13821 13822 13823 13824 13825 13826 13827 13828 13829 13830 13831 13832 13833 13834 13835 13836 13837 13838 13839 13840 13841 13842 13843 13844 13845 13846 13847 13848 13849 13850 13851 13852 13853 13854 13855 13856 13857 13858 13859 13860 13861 13862 13863 13864 13865 13866 13867 13868 13869 13870 13871 13872 13873 13874 13875 13876 13877 13878 13879 13880 13881 13882 13883 13884 13885 13886 13887 13888 13889 13890 13891 13892 13893 13894 13895 13896 13897 13898 13899 13900 13901 13902 13903 13904 13905 13906 13907 13908 13909 13910 13911 13912 13913 13914 13915 13916 13917 13918 13919 13920 13921 13922 13923 13924 13925 13926 13927 13928 13929 13930 13931 13932 13933 13934 13935 13936 13937 13938 13939 13940 13941 13942 13943 13944 13945 13946 13947 13948 13949 13950 13951 13952 13953 13954 13955 13956 13957 13958 13959 13960 13961 13962 13963 13964 13965 13966 13967 13968 13969 13970 13971 13972 13973 13974 13975 13976 13977 13978 13979 13980 13981 13982 13983 13984 13985 13986 13987 13988 13989 13990 13991 13992 13993 13994 13995 13996 13997 13998 13999 14000 14001 14002 14003 14004 14005 14006 14007 14008 14009 14010 14011 14012 14013 14014 14015 14016 14017 14018 14019 14020 14021 14022 14023 14024 14025 14026 14027 14028 14029 14030 14031 14032 14033 14034 14035 14036 14037 14038 14039 14040 14041 14042 14043 14044 14045 14046 14047 14048 14049 14050 14051 14052 14053 14054 14055 14056 14057 14058 14059 14060 14061 14062 14063 14064 14065 14066 14067 14068 14069 14070 14071 14072 14073 14074 14075 14076 14077 14078 14079 14080 14081 14082 14083 14084 14085 14086 14087 14088 14089 14090 14091 14092 14093 14094 14095 14096 14097 14098 14099 14100 14101 14102 14103 14104 14105 14106 14107 14108 14109 14110 14111 14112 14113 14114 14115 14116 14117 14118 14119 14120 14121 14122 14123 14124 14125 14126 14127 14128 14129 14130 14131 14132 14133 14134 14135 14136 14137 14138 14139 14140 14141 14142 14143 14144 14145 14146 14147 14148 14149 14150 14151 14152 14153 14154 14155 14156 14157 14158 14159 14160 14161 14162 14163 14164 14165 14166 14167 14168 14169 14170 14171 14172 14173 14174 14175 14176 14177 14178 14179 14180 14181 14182 14183 14184 14185 14186 14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 14210 14211 14212 14213 14214 14215 14216 14217 14218 14219 14220 14221 14222 14223 14224 14225 14226 14227 14228 14229 14230 14231 14232 14233 14234 14235 14236 14237 14238 14239 14240 14241 14242 14243 14244 14245 14246 14247 14248 14249 14250 14251 14252 14253 14254 14255 14256 14257 14258 14259 14260 14261 14262 14263 14264 14265 14266 14267 14268 14269 14270 14271 14272 14273 14274 14275 14276 14277 14278 14279 14280 14281 14282 14283 14284 14285 14286 14287 14288 14289 14290 14291 14292 14293 14294 14295 14296 14297 14298 14299 14300 14301 14302 14303 14304 14305 14306 14307 14308 14309 14310 14311 14312 14313 14314 14315 14316 14317 14318 14319 14320 14321 14322 14323 14324 14325 14326 14327 14328 14329 14330 14331 14332 14333 14334 14335 14336 14337 14338 14339 14340 14341 14342 14343 14344 14345 14346 14347 14348 14349 14350 14351 14352 14353 14354 14355 14356 14357 14358 14359 14360 14361 14362 14363 14364 14365 14366 14367 14368 14369 14370 14371 14372 14373 14374 14375 14376 14377 14378 14379 14380 14381 14382 14383 14384 14385 14386 14387 14388 14389 14390 14391 14392 14393 14394 14395 14396 14397 14398 14399 14400 14401 14402 14403 14404 14405 14406 14407 14408 14409 14410 14411 14412 14413 14414 14415 14416 14417 14418 14419 14420 14421 14422 14423 14424 14425 14426 14427 14428 14429 14430 14431 14432 14433 14434 14435 14436 14437 14438 14439 14440 14441 14442 14443 14444 14445 14446 14447 14448 14449 14450 14451 14452 14453 14454 14455 14456 14457 14458 14459 14460 14461 14462 14463 14464 14465 14466 14467 14468 14469 14470 14471 14472 14473 14474 14475 14476 14477 14478 14479 14480 14481 14482 14483 14484 14485 14486 14487 14488 14489 14490 14491 14492 14493 14494 14495 14496 14497 14498 14499 14500 14501 14502 14503 14504 14505 14506 14507 14508 14509 14510 14511 14512 14513 14514 14515 14516 14517 14518 14519 14520 14521 14522 14523 14524 14525 14526 14527 14528 14529 14530 14531 14532 14533 14534 14535 14536 14537 14538 14539 14540 14541 14542 14543 14544 14545 14546 14547 14548 14549 14550 14551 14552 14553 14554 14555 14556 14557 14558 14559 14560 14561 14562 14563 14564 14565 14566 14567 14568 14569 14570 14571 14572 14573 14574 14575 14576 14577 14578 14579 14580 14581 14582 14583 14584 14585 14586 14587 14588 14589 14590 14591 14592 14593 14594 14595 14596 14597 14598 14599 14600 14601 14602 14603 14604 14605 14606 14607 14608 14609 14610 14611 14612 14613 14614 14615 14616 14617 14618 14619 14620 14621 14622 14623 14624 14625 14626 14627 14628 14629 14630 14631 14632 14633 14634 14635 14636 14637 14638 14639 14640 14641 14642 14643 14644 14645 14646 14647 14648 14649 14650 14651 14652 14653 14654 14655 14656 14657 14658 14659 14660 14661 14662 14663 14664 14665 14666 14667 14668 14669 14670 14671 14672 14673 14674 14675 14676 14677 14678 14679 14680 14681 14682 14683 14684 14685 14686 14687 14688 14689 14690 14691 14692 14693 14694 14695 14696 14697 14698 14699 14700 14701 14702 14703 14704 14705 14706 14707 14708 14709 14710 14711 14712 14713 14714 14715 14716 14717 14718 14719 14720 14721 14722 14723 14724 14725 14726 14727 14728 14729 14730 14731 14732 14733 14734 14735 14736 14737 14738 14739 14740 14741 14742 14743 14744 14745 14746 14747 14748 14749 14750 14751 14752 14753 14754 14755 14756 14757 14758 14759 14760 14761 14762 14763 14764 14765 14766 14767 14768 14769 14770 14771 14772 14773 14774 14775 14776 14777 14778 14779 14780 14781 14782 14783 14784 14785 14786 14787 14788 14789 14790 14791 14792 14793 14794 14795 14796 14797 14798 14799 14800 14801 14802 14803 14804 14805 14806 14807 14808 14809 14810 14811 14812 14813 14814 14815 14816 14817 14818 14819 14820 14821 14822 14823 14824 14825 14826 14827 14828 14829 14830 14831 14832 14833 14834 14835 14836 14837 14838 14839 14840 14841 14842 14843 14844 14845 14846 14847 14848 14849 14850 14851 14852 14853 14854 14855 14856 14857 14858 14859 14860 14861 14862 14863 14864 14865 14866 14867 14868 14869 14870 14871 14872 14873 14874 14875 14876 14877 14878 14879 14880 14881 14882 14883 14884 14885 14886 14887 14888 14889 14890 14891 14892 14893 14894 14895 14896 14897 14898 14899 14900 14901 14902 14903 14904 14905 14906 14907 14908 14909 14910 14911 14912 14913 14914 14915 14916 14917 14918 14919 14920 14921 14922 14923 14924 14925 14926 14927 14928 14929 14930 14931 14932 14933 14934 14935 14936 14937 14938 14939 14940 14941 14942 14943 14944 14945 14946 14947 14948 14949 14950 14951 14952 14953 14954 14955 14956 14957 14958 14959 14960 14961 14962 14963 14964 14965 14966 14967 14968 14969 14970 14971 14972 14973 14974 14975 14976 14977 14978 14979 14980 14981 14982 14983 14984 14985 14986 14987 14988 14989 14990 14991 14992 14993 14994 14995 14996 14997 14998 14999 15000 15001 15002 15003 15004 15005 15006 15007 15008 15009 15010 15011 15012 15013 15014 15015 15016 15017 15018 15019 15020 15021 15022 15023 15024 15025 15026 15027 15028 15029 15030 15031 15032 15033 15034 15035 15036 15037 15038 15039 15040 15041 15042 15043 15044 15045 15046 15047 15048 15049 15050 15051 15052 15053 15054 15055 15056 15057 15058 15059 15060 15061 15062 15063 15064 15065 15066 15067 15068 15069 15070 15071 15072 15073 15074 15075 15076 15077 15078 15079 15080 15081 15082 15083 15084 15085 15086 15087 15088 15089 15090 15091 15092 15093 15094 15095 15096 15097 15098 15099 15100 15101 15102 15103 15104 15105 15106 15107 15108 15109 15110 15111 15112 15113 15114 15115 15116 15117 15118 15119 15120 15121 15122 15123 15124 15125 15126 15127 15128 15129 15130 15131 15132 15133 15134 15135 15136 15137 15138 15139 15140 15141 15142 15143 15144 15145 15146 15147 15148 15149 15150 15151 15152 15153 15154 15155 15156 15157 15158 15159 15160 15161 15162 15163 15164 15165 15166 15167 15168 15169 15170 15171 15172 15173 15174 15175 15176 15177 15178 15179 15180 15181 15182 15183 15184 15185 15186 15187 15188 15189 15190 15191 15192 15193 15194 15195 15196 15197 15198 15199 15200 15201 15202 15203 15204 15205 15206 15207 15208 15209 15210 15211 15212 15213 15214 15215 15216 15217 15218 15219 15220 15221 15222 15223 15224 15225 15226 15227 15228 15229 15230 15231 15232 15233 15234 15235 15236 15237 15238 15239 15240 15241 15242 15243 15244 15245 15246 15247 15248 15249 15250 15251 15252 15253 15254 15255 15256 15257 15258 15259 15260 15261 15262 15263 15264 15265 15266 15267 15268 15269 15270 15271 15272 15273 15274 15275 15276 15277 15278 15279 15280 15281 15282 15283 15284 15285 15286 15287 15288 15289 15290 15291 15292 15293 15294 15295 15296 15297 15298 15299 15300 15301 15302 15303 15304 15305 15306 15307 15308 15309 15310 15311 15312 15313 15314 15315 15316 15317 15318 15319 15320 15321 15322 15323 15324 15325 15326 15327 15328 15329 15330 15331 15332 15333 15334 15335 15336 15337 15338 15339 15340 15341 15342 15343 15344 15345 15346 15347 15348 15349 15350 15351 15352 15353 15354 15355 15356 15357 15358 15359 15360 15361 15362 15363 15364 15365 15366 15367 15368 15369 15370 15371 15372 15373 15374 15375 15376 15377 15378 15379 15380 15381 15382 15383 15384 15385 15386 15387 15388 15389 15390 15391 15392 15393 15394 15395 15396 15397 15398 15399 15400 15401 15402 15403 15404 15405 15406 15407 15408 15409 15410 15411 15412 15413 15414 15415 15416 15417 15418 15419 15420 15421 15422 15423 15424 15425 15426 15427 15428 15429 15430 15431 15432 15433 15434 15435 15436 15437 15438 15439 15440 15441 15442 15443 15444 15445 15446 15447 15448 15449 15450 15451 15452 15453 15454 15455 15456 15457 15458 15459 15460 15461 15462 15463 15464 15465 15466 15467 15468 15469 15470 15471 15472 15473 15474 15475 15476 15477 15478 15479 15480 15481 15482 15483 15484 15485 15486 15487 15488 15489 15490 15491 15492 15493 15494 15495 15496 15497 15498 15499 15500 15501 15502 15503 15504 15505 15506 15507 15508 15509 15510 15511 15512 15513 15514 15515 15516 15517 15518 15519 15520 15521 15522 15523 15524 15525 15526 15527 15528 15529 15530 15531 15532 15533 15534 15535 15536 15537 15538 15539 15540 15541 15542 15543 15544 15545 15546 15547 15548 15549 15550 15551 15552 15553 15554 15555 15556 15557 15558 15559 15560 15561 15562 15563 15564 15565 15566 15567 15568 15569 15570 15571 15572 15573 15574 15575 15576 15577 15578 15579 15580 15581 15582 15583 15584 15585 15586 15587 15588 15589 15590 15591 15592 15593 15594 15595 15596 15597 15598 15599 15600 15601 15602 15603 15604 15605 15606 15607 15608 15609 15610 15611 15612 15613 15614 15615 15616 15617 15618 15619 15620 15621 15622 15623 15624 15625 15626 15627 15628 15629 15630 15631 15632 15633 15634 15635 15636 15637 15638 15639 15640 15641 15642 15643 15644 15645 15646 15647 15648 15649 15650 15651 15652 15653 15654 15655 15656 15657 15658 15659 15660 15661 15662 15663 15664 15665 15666 15667 15668 15669 15670 15671 15672 15673 15674 15675 15676 15677 15678 15679 15680 15681 15682 15683 15684 15685 15686 15687 15688 15689 15690 15691 15692 15693 15694 15695 15696 15697 15698 15699 15700 15701 15702 15703 15704 15705 15706 15707 15708 15709 15710 15711 15712 15713 15714 15715 15716 15717 15718 15719 15720 15721 15722 15723 15724 15725 15726 15727 15728 15729 15730 15731 15732 15733 15734 15735 15736 15737 15738 15739 15740 15741 15742 15743 15744 15745 15746 15747 15748 15749 15750 15751 15752 15753 15754 15755 15756 15757 15758 15759 15760 15761 15762 15763 15764 15765 15766 15767 15768 15769 15770 15771 15772 15773 15774 15775 15776 15777 15778 15779 15780 15781 15782 15783 15784 15785 15786 15787 15788 15789 15790 15791 15792 15793 15794 15795 15796 15797 15798 15799 15800 15801 15802 15803 15804 15805 15806 15807 15808 15809 15810 15811 15812 15813 15814 15815 15816 15817 15818 15819 15820 15821 15822 15823 15824 15825 15826 15827 15828 15829 15830 15831 15832 15833 15834 15835 15836 15837 15838 15839 15840 15841 15842 15843 15844 15845 15846 15847 15848 15849 15850 15851 15852 15853 15854 15855 15856 15857 15858 15859 15860 15861 15862 15863 15864 15865 15866 15867 15868 15869 15870 15871 15872 15873 15874 15875 15876 15877 15878 15879 15880 15881 15882 15883 15884 15885 15886 15887 15888 15889 15890 15891 15892 15893 15894 15895 15896 15897 15898 15899 15900 15901 15902 15903 15904 15905 15906 15907 15908 15909 15910 15911 15912 15913 15914 15915 15916 15917 15918 15919 15920 15921 15922 15923 15924 15925 15926 15927 15928 15929 15930 15931 15932 15933 15934 15935 15936 15937 15938 15939 15940 15941 15942 15943 15944 15945 15946 15947 15948 15949 15950 15951 15952 15953 15954 15955 15956 15957 15958 15959 15960 15961 15962 15963 15964 15965 15966 15967 15968 15969 15970 15971 15972 15973 15974 15975 15976 15977 15978 15979 15980 15981 15982 15983 15984 15985 15986 15987 15988 15989 15990 15991 15992 15993 15994 15995 15996 15997 15998 15999 16000 16001 16002 16003 16004 16005 16006 16007 16008 16009 16010 16011 16012 16013 16014 16015 16016 16017 16018 16019 16020 16021 16022 16023 16024 16025 16026 16027 16028 16029 16030 16031 16032 16033 16034 16035 16036 16037 16038 16039 16040 16041 16042 16043 16044 16045 16046 16047 16048 16049 16050 16051 16052 16053 16054 16055 16056 16057 16058 16059 16060 16061 16062 16063 16064 16065 16066 16067 16068 16069 16070 16071 16072 16073 16074 16075 16076 16077 16078 16079 16080 16081 16082 16083 16084 16085 16086 16087 16088 16089 16090 16091 16092 16093 16094 16095 16096 16097 16098 16099 16100 16101 16102 16103 16104 16105 16106 16107 16108 16109 16110 16111 16112 16113 16114 16115 16116 16117 16118 16119 16120 16121 16122 16123 16124 16125 16126 16127 16128 16129 16130 16131 16132 16133 16134 16135 16136 16137 16138 16139 16140 16141 16142 16143 16144 16145 16146 16147 16148 16149 16150 16151 16152 16153 16154 16155 16156 16157 16158 16159 16160 16161 16162 16163 16164 16165 16166 16167 16168 16169 16170 16171 16172 16173 16174 16175 16176 16177 16178 16179 16180 16181 16182 16183 16184 16185 16186 16187 16188 16189 16190 16191 16192 16193 16194 16195 16196 16197 16198 16199 16200 16201 16202 16203 16204 16205 16206 16207 16208 16209 16210 16211 16212 16213 16214 16215 16216 16217 16218 16219 16220 16221 16222 16223 16224 16225 16226 16227 16228 16229 16230 16231 16232 16233 16234 16235 16236 16237 16238 16239 16240 16241 16242 16243 16244 16245 16246 16247 16248 16249 16250 16251 16252 16253 16254 16255 16256 16257 16258 16259 16260 16261 16262 16263 16264 16265 16266 16267 16268 16269 16270 16271 16272 16273 16274 16275 16276 16277 16278 16279 16280 16281 16282 16283 16284 16285 16286 16287 16288 16289 16290 16291 16292 16293 16294 16295 16296 16297 16298 16299 16300 16301 16302 16303 16304 16305 16306 16307 16308 16309 16310 16311 16312 16313 16314 16315 16316 16317 16318 16319 16320 16321 16322 16323 16324 16325 16326 16327 16328 16329 16330 16331 16332 16333 16334 16335 16336 16337 16338 16339 16340 16341 16342 16343 16344 16345 16346 16347 16348 16349 16350 16351 16352 16353 16354 16355 16356 16357 16358 16359 16360 16361 16362 16363 16364 16365 16366 16367 16368 16369 16370 16371 16372 16373 16374 16375 16376 16377 16378 16379 16380 16381 16382 16383 16384 16385 16386 16387 16388 16389 16390 16391 16392 16393 16394 16395 16396 16397 16398 16399 16400 16401 16402 16403 16404 16405 16406 16407 16408 16409 16410 16411 16412 16413 16414 16415 16416 16417 16418 16419 16420 16421 16422 16423 16424 16425 16426 16427 16428 16429 16430 16431 16432 16433 16434 16435 16436 16437 16438 16439 16440 16441 16442 16443 16444 16445 16446 16447 16448 16449 16450 16451 16452 16453 16454 16455 16456 16457 16458 16459 16460 16461 16462 16463 16464 16465 16466 16467 16468 16469 16470 16471 16472 16473 16474 16475 16476 16477 16478 16479 16480 16481 16482 16483 16484 16485 16486 16487 16488 16489 16490 16491 16492 16493 16494 16495 16496 16497 16498 16499 16500 16501 16502 16503 16504 16505 16506 16507 16508 16509 16510 16511 16512 16513 16514 16515 16516 16517 16518 16519 16520 16521 16522 16523 16524 16525 16526 16527 16528 16529 16530 16531 16532 16533 16534 16535 16536 16537 16538 16539 16540 16541 16542 16543 16544 16545 16546 16547 16548 16549 16550 16551 16552 16553 16554 16555 16556 16557 16558 16559 16560 16561 16562 16563 16564 16565 16566 16567 16568 16569 16570 16571 16572 16573 16574 16575 16576 16577 16578 16579 16580 16581 16582 16583 16584 16585 16586 16587 16588 16589 16590 16591 16592 16593 16594 16595 16596 16597 16598 16599 16600 16601 16602 16603 16604 16605 16606 16607 16608 16609 16610 16611 16612 16613 16614 16615 16616 16617 16618 16619 16620 16621 16622 16623 16624 16625 16626 16627 16628 16629 16630 16631 16632 16633 16634 16635 16636 16637 16638 16639 16640 16641 16642 16643 16644 16645 16646 16647 16648 16649 16650 16651 16652 16653 16654 16655 16656 16657 16658 16659 16660 16661 16662 16663 16664 16665 16666 16667 16668 16669 16670 16671 16672 16673 16674 16675 16676 16677 16678 16679 16680 16681 16682 16683 16684 16685 16686 16687 16688 16689 16690 16691 16692 16693 16694 16695 16696 16697 16698 16699 16700 16701 16702 16703 16704 16705 16706 16707 16708 16709 16710 16711 16712 16713 16714 16715 16716 16717 16718 16719 16720 16721 16722 16723 16724 16725 16726 16727 16728 16729 16730 16731 16732 16733 16734 16735 16736 16737 16738 16739 16740 16741 16742 16743 16744 16745 16746 16747 16748 16749 16750 16751 16752 16753 16754 16755 16756 16757 16758 16759 16760 16761 16762 16763 16764 16765 16766 16767 16768 16769 16770 16771 16772 16773 16774 16775 16776 16777 16778 16779 16780 16781 16782 16783 16784 16785 16786 16787 16788 16789 16790 16791 16792 16793 16794 16795 16796 16797 16798 16799 16800 16801 16802 16803 16804 16805 16806 16807 16808 16809 16810 16811 16812 16813 16814 16815 16816 16817 16818 16819 16820 16821 16822 16823 16824 16825 16826 16827 16828 16829 16830 16831 16832 16833 16834 16835 16836 16837 16838 16839 16840 16841 16842 16843 16844 16845 16846 16847 16848 16849 16850 16851 16852 16853 16854 16855 16856 16857 16858 16859 16860 16861 16862 16863 16864 16865 16866 16867 16868 16869 16870 16871 16872 16873 16874 16875 16876 16877 16878 16879 16880 16881 16882 16883 16884 16885 16886 16887 16888 16889 16890 16891 16892 16893 16894 16895 16896 16897 16898 16899 16900 16901 16902 16903 16904 16905 16906 16907 16908 16909 16910 16911 16912 16913 16914 16915 16916 16917 16918 16919 16920 16921 16922 16923 16924 16925 16926 16927 16928 16929 16930 16931 16932 16933 16934 16935 16936 16937 16938 16939 16940 16941 16942 16943 16944 16945 16946 16947 16948 16949 16950 16951 16952 16953 16954 16955 16956 16957 16958 16959 16960 16961 16962 16963 16964 16965 16966 16967 16968 16969 16970 16971 16972 16973 16974 16975 16976 16977 16978 16979 16980 16981 16982 16983 16984 16985 16986 16987 16988 16989 16990 16991 16992 16993 16994 16995 16996 16997 16998 16999 17000 17001 17002 17003 17004 17005 17006 17007 17008 17009 17010 17011 17012 17013 17014 17015 17016 17017 17018 17019 17020 17021 17022 17023 17024 17025 17026 17027 17028 17029 17030 17031 17032 17033 17034 17035 17036 17037 17038 17039 17040 17041 17042 17043 17044 17045 17046 17047 17048 17049 17050 17051 17052 17053 17054 17055 17056 17057 17058 17059 17060 17061 17062 17063 17064 17065 17066 17067 17068 17069 17070 17071 17072 17073 17074 17075 17076 17077 17078 17079 17080 17081 17082 17083 17084 17085 17086 17087 17088 17089 17090 17091 17092 17093 17094 17095 17096 17097 17098 17099 17100 17101 17102 17103 17104 17105 17106 17107 17108 17109 17110 17111 17112 17113 17114 17115 17116 17117 17118 17119 17120 17121 17122 17123 17124 17125 17126 17127 17128 17129 17130 17131 17132 17133 17134 17135 17136 17137 17138 17139 17140 17141 17142 17143 17144 17145 17146 17147 17148 17149 17150 17151 17152 17153 17154 17155 17156 17157 17158 17159 17160 17161 17162 17163 17164 17165 17166 17167 17168 17169 17170 17171 17172 17173 17174 17175 17176 17177 17178 17179 17180 17181 17182 17183 17184 17185 17186 17187 17188 17189 17190 17191 17192 17193 17194 17195 17196 17197 17198 17199 17200 17201 17202 17203 17204 17205 17206 17207 17208 17209 17210 17211 17212 17213 17214 17215 17216 17217 17218 17219 17220 17221 17222 17223 17224 17225 17226 17227 17228 17229 17230 17231 17232 17233 17234 17235 17236 17237 17238 17239 17240 17241 17242 17243 17244 17245 17246 17247 17248 17249 17250 17251 17252 17253 17254 17255 17256 17257 17258 17259 17260 17261 17262 17263 17264 17265 17266 17267 17268 17269 17270 17271 17272 17273 17274 17275 17276 17277 17278 17279 17280 17281 17282 17283 17284 17285 17286 17287 17288 17289 17290 17291 17292 17293 17294 17295 17296 17297 17298 17299 17300 17301 17302 17303 17304 17305 17306 17307 17308 17309 17310 17311 17312 17313 17314 17315 17316 17317 17318 17319 17320 17321 17322 17323 17324 17325 17326 17327 17328 17329 17330 17331 17332 17333 17334 17335 17336 17337 17338 17339 17340 17341 17342 17343 17344 17345 17346 17347 17348 17349 17350 17351 17352 17353 17354 17355 17356 17357 17358 17359 17360 17361 17362 17363 17364 17365 17366 17367 17368 17369 17370 17371 17372 17373 17374 17375 17376 17377 17378 17379 17380 17381 17382 17383 17384 17385 17386 17387 17388 17389 17390 17391 17392 17393 17394 17395 17396 17397 17398 17399 17400 17401 17402 17403 17404 17405 17406 17407 17408 17409 17410 17411 17412 17413 17414 17415 17416 17417 17418 17419 17420 17421 17422 17423 17424 17425 17426 17427 17428 17429 17430 17431 17432 17433 17434 17435 17436 17437 17438 17439 17440 17441 17442 17443 17444 17445 17446 17447 17448 17449 17450 17451 17452 17453 17454 17455 17456 17457 17458 17459 17460 17461 17462 17463 17464 17465 17466 17467 17468 17469 17470 17471 17472 17473 17474 17475 17476 17477 17478 17479 17480 17481 17482 17483 17484 17485 17486 17487 17488 17489 17490 17491 17492 17493 17494 17495 17496 17497 17498 17499 17500 17501 17502 17503 17504 17505 17506 17507 17508 17509 17510 17511 17512 17513 17514 17515 17516 17517 17518 17519 17520 17521 17522 17523 17524 17525 17526 17527 17528 17529 17530 17531 17532 17533 17534 17535 17536 17537 17538 17539 17540 17541 17542 17543 17544 17545 17546 17547 17548 17549 17550 17551 17552 17553 17554 17555 17556 17557 17558 17559 17560 17561 17562 17563 17564 17565 17566 17567 17568 17569 17570 17571 17572 17573 17574 17575 17576 17577 17578 17579 17580 17581 17582 17583 17584 17585 17586 17587 17588 17589 17590 17591 17592 17593 17594 17595 17596 17597 17598 17599 17600 17601 17602 17603 17604 17605 17606 17607 17608 17609 17610 17611 17612 17613 17614 17615 17616 17617 17618 17619 17620 17621 17622 17623 17624 17625 17626 17627 17628 17629 17630 17631 17632 17633 17634 17635 17636 17637 17638 17639 17640 17641 17642 17643 17644 17645 17646 17647 17648 17649 17650 17651 17652 17653 17654 17655 17656 17657 17658 17659 17660 17661 17662 17663 17664 17665 17666 17667 17668 17669 17670 17671 17672 17673 17674 17675 17676 17677 17678 17679 17680 17681 17682 17683 17684 17685 17686 17687 17688 17689 17690 17691 17692 17693 17694 17695 17696 17697 17698 17699 17700 17701 17702 17703 17704 17705 17706 17707 17708 17709 17710 17711 17712 17713 17714 17715 17716 17717 17718 17719 17720 17721 17722 17723 17724 17725 17726 17727 17728 17729 17730 17731 17732 17733 17734 17735 17736 17737 17738 17739 17740 17741 17742 17743 17744 17745 17746 17747 17748 17749 17750 17751 17752 17753 17754 17755 17756 17757 17758 17759 17760 17761 17762 17763 17764 17765 17766 17767 17768 17769 17770 17771 17772 17773 17774 17775 17776 17777 17778 17779 17780 17781 17782 17783 17784 17785 17786 17787 17788 17789 17790 17791 17792 17793 17794 17795 17796 17797 17798 17799 17800 17801 17802 17803 17804 17805 17806 17807 17808 17809 17810 17811 17812 17813 17814 17815 17816 17817 17818 17819 17820 17821 17822 17823 17824 17825 17826 17827 17828 17829 17830 17831 17832 17833 17834 17835 17836 17837 17838 17839 17840 17841 17842 17843 17844 17845 17846 17847 17848 17849 17850 17851 17852 17853 17854 17855 17856 17857 17858 17859 17860 17861 17862 17863 17864 17865 17866 17867 17868 17869 17870 17871 17872 17873 17874 17875 17876 17877 17878 17879 17880 17881 17882 17883 17884 17885 17886 17887 17888 17889 17890 17891 17892 17893 17894 17895 17896 17897 17898 17899 17900 17901 17902 17903 17904 17905 17906 17907 17908 17909 17910 17911 17912 17913 17914 17915 17916 17917 17918 17919 17920 17921 17922 17923 17924 17925 17926 17927 17928 17929 17930 17931 17932 17933 17934 17935 17936 17937 17938 17939 17940 17941 17942 17943 17944 17945 17946 17947 17948 17949 17950 17951 17952 17953 17954 17955 17956 17957 17958 17959 17960 17961 17962 17963 17964 17965 17966 17967 17968 17969 17970 17971 17972 17973 17974 17975 17976 17977 17978 17979 17980 17981 17982 17983 17984 17985 17986 17987 17988 17989 17990 17991 17992 17993 17994 17995 17996 17997 17998 17999 18000 18001 18002 18003 18004 18005 18006 18007 18008 18009 18010 18011 18012 18013 18014 18015 18016 18017 18018 18019 18020 18021 18022 18023 18024 18025 18026 18027 18028 18029 18030 18031 18032 18033 18034 18035 18036 18037 18038 18039 18040 18041 18042 18043 18044 18045 18046 18047 18048 18049 18050 18051 18052 18053 18054 18055 18056 18057 18058 18059 18060 18061 18062 18063 18064 18065 18066 18067 18068 18069 18070 18071 18072 18073 18074 18075 18076 18077 18078 18079 18080 18081 18082 18083 18084 18085 18086 18087 18088 18089 18090 18091 18092 18093 18094 18095 18096 18097 18098 18099 18100 18101 18102 18103 18104 18105 18106 18107 18108 18109 18110 18111 18112 18113 18114 18115 18116 18117 18118 18119 18120 18121 18122 18123 18124 18125 18126 18127 18128 18129 18130 18131 18132 18133 18134 18135 18136 18137 18138 18139 18140 18141 18142 18143 18144 18145 18146 18147 18148 18149 18150 18151 18152 18153 18154 18155 18156 18157 18158 18159 18160 18161 18162 18163 18164 18165 18166 18167 18168 18169 18170 18171 18172 18173 18174 18175 18176 18177 18178 18179 18180 18181 18182 18183 18184 18185 18186 18187 18188 18189 18190 18191 18192 18193 18194 18195 18196 18197 18198 18199 18200 18201 18202 18203 18204 18205 18206 18207 18208 18209 18210 18211 18212 18213 18214 18215 18216 18217 18218 18219 18220 18221 18222 18223 18224 18225 18226 18227 18228 18229 18230 18231 18232 18233 18234 18235 18236 18237 18238 18239 18240 18241 18242 18243 18244 18245 18246 18247 18248 18249 18250 18251 18252 18253 18254 18255 18256 18257 18258 18259 18260 18261 18262 18263 18264 18265 18266 18267 18268 18269 18270 18271 18272 18273 18274 18275 18276 18277 18278 18279 18280 18281 18282 18283 18284 18285 18286 18287 18288 18289 18290 18291 18292 18293 18294 18295 18296 18297 18298 18299 18300 18301 18302 18303 18304 18305 18306 18307 18308 18309 18310 18311 18312 18313 18314 18315 18316 18317 18318 18319 18320 18321 18322 18323 18324 18325 18326 18327 18328 18329 18330 18331 18332 18333 18334 18335 18336 18337 18338 18339 18340 18341 18342 18343 18344 18345 18346 18347 18348 18349 18350 18351 18352 18353 18354 18355 18356 18357 18358 18359 18360 18361 18362 18363 18364 18365 18366 18367 18368 18369 18370 18371 18372 18373 18374 18375 18376 18377 18378 18379 18380 18381 18382 18383 18384 18385 18386 18387 18388 18389 18390 18391 18392 18393 18394 18395 18396 18397 18398 18399 18400 18401 18402 18403 18404 18405 18406 18407 18408 18409 18410 18411 18412 18413 18414 18415 18416 18417 18418 18419 18420 18421 18422 18423 18424 18425 18426 18427 18428 18429 18430 18431 18432 18433 18434 18435 18436 18437 18438 18439 18440 18441 18442 18443 18444 18445 18446 18447 18448 18449 18450 18451 18452 18453 18454 18455 18456 18457 18458 18459 18460 18461 18462 18463 18464 18465 18466 18467 18468 18469 18470 18471 18472 18473 18474 18475 18476 18477 18478 18479 18480 18481 18482 18483 18484 18485 18486 18487 18488 18489 18490 18491 18492 18493 18494 18495 18496 18497 18498 18499 18500 18501 18502 18503 18504 18505 18506 18507 18508 18509 18510 18511 18512 18513 18514 18515 18516 18517 18518 18519 18520 18521 18522 18523 18524 18525 18526 18527 18528 18529 18530 18531 18532 18533 18534 18535 18536 18537 18538 18539 18540 18541 18542 18543 18544 18545 18546 18547 18548 18549 18550 18551 18552 18553 18554 18555 18556 18557 18558 18559 18560 18561 18562 18563 18564 18565 18566 18567 18568 18569 18570 18571 18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 18595 18596 18597 18598 18599 18600 18601 18602 18603 18604 18605 18606 18607 18608 18609 18610 18611 18612 18613 18614 18615 18616 18617 18618 18619 18620 18621 18622 18623 18624 18625 18626 18627 18628 18629 18630 18631 18632 18633 18634 18635 18636 18637 18638 18639 18640 18641 18642 18643 18644 18645 18646 18647 18648 18649 18650 18651 18652 18653 18654 18655 18656 18657 18658 18659 18660 18661 18662 18663 18664 18665 18666 18667 18668 18669 18670 18671 18672 18673 18674 18675 18676 18677 18678 18679 18680 18681 18682 18683 18684 18685 18686 18687 18688 18689 18690 18691 18692 18693 18694 18695 18696 18697 18698 18699 18700 18701 18702 18703 18704 18705 18706 18707 18708 18709 18710 18711 18712 18713 18714 18715 18716 18717 18718 18719 18720 18721 18722 18723 18724 18725 18726 18727 18728 18729 18730 18731 18732 18733 18734 18735 18736 18737 18738 18739 18740 18741 18742 18743 18744 18745 18746 18747 18748 18749 18750 18751 18752 18753 18754 18755 18756 18757 18758 18759 18760 18761 18762 18763 18764 18765 18766 18767 18768 18769 18770 18771 18772 18773 18774 18775 18776 18777 18778 18779 18780 18781 18782 18783 18784 18785 18786 18787 18788 18789 18790 18791 18792 18793 18794 18795 18796 18797 18798 18799 18800 18801 18802 18803 18804 18805 18806 18807 18808 18809 18810 18811 18812 18813 18814 18815 18816 18817 18818 18819 18820 18821 18822 18823 18824 18825 18826 18827 18828 18829 18830 18831 18832 18833 18834 18835 18836 18837 18838 18839 18840 18841 18842 18843 18844 18845 18846 18847 18848 18849 18850 18851 18852 18853 18854 18855 18856 18857 18858 18859 18860 18861 18862 18863 18864 18865 18866 18867 18868 18869 18870 18871 18872 18873 18874 18875 18876 18877 18878 18879 18880 18881 18882 18883 18884 18885 18886 18887 18888 18889 18890 18891 18892 18893 18894 18895 18896 18897 18898 18899 18900 18901 18902 18903 18904 18905 18906 18907 18908 18909 18910 18911 18912 18913 18914 18915 18916 18917 18918 18919 18920 18921 18922 18923 18924 18925 18926 18927 18928 18929 18930 18931 18932 18933 18934 18935 18936 18937 18938 18939 18940 18941 18942 18943 18944 18945 18946 18947 18948 18949 18950 18951 18952 18953 18954 18955 18956 18957 18958 18959 18960 18961 18962 18963 18964 18965 18966 18967 18968 18969 18970 18971 18972 18973 18974 18975 18976 18977 18978 18979 18980 18981 18982 18983 18984 18985 18986 18987 18988 18989 18990 18991 18992 18993 18994 18995 18996 18997 18998 18999 19000 19001 19002 19003 19004 19005 19006 19007 19008 19009 19010 19011 19012 19013 19014 19015 19016 19017 19018 19019 19020 19021 19022 19023 19024 19025 19026 19027 19028 19029 19030 19031 19032 19033 19034 19035 19036 19037 19038 19039 19040 19041 19042 19043 19044 19045 19046 19047 19048 19049 19050 19051 19052 19053 19054 19055 19056 19057 19058 19059 19060 19061 19062 19063 19064 19065 19066 19067 19068 19069 19070 19071 19072 19073 19074 19075 19076 19077 19078 19079 19080 19081 19082 19083 19084 19085 19086 19087 19088 19089 19090 19091 19092 19093 19094 19095 19096 19097 19098 19099 19100 19101 19102 19103 19104 19105 19106 19107 19108 19109 19110 19111 19112 19113 19114 19115 19116 19117 19118 19119 19120 19121 19122 19123 19124 19125 19126 19127 19128 19129 19130 19131 19132 19133 19134 19135 19136 19137 19138 19139 19140 19141 19142 19143 19144 19145 19146 19147 19148 19149 19150 19151 19152 19153 19154 19155 19156 19157 19158 19159 19160 19161 19162 19163 19164 19165 19166 19167 19168 19169 19170 19171 19172 19173 19174 19175 19176 19177 19178 19179 19180 19181 19182 19183 19184 19185 19186 19187 19188 19189 19190 19191 19192 19193 19194 19195 19196 19197 19198 19199 19200 19201 19202 19203 19204 19205 19206 19207 19208 19209 19210 19211 19212 19213 19214 19215 19216 19217 19218 19219 19220 19221 19222 19223 19224 19225 19226 19227 19228 19229 19230 19231 19232 19233 19234 19235 19236 19237 19238 19239 19240 19241 19242 19243 19244 19245 19246 19247 19248 19249 19250 19251 19252 19253 19254 19255 19256 19257 19258 19259 19260 19261 19262 19263 19264 19265 19266 19267 19268 19269 19270 19271 19272 19273 19274 19275 19276 19277 19278 19279 19280 19281 19282 19283 19284 19285 19286 19287 19288 19289 19290 19291 19292 19293 19294 19295 19296 19297 19298 19299 19300 19301 19302 19303 19304 19305 19306 19307 19308 19309 19310 19311 19312 19313 19314 19315 19316 19317 19318 19319 19320 19321 19322 19323 19324 19325 19326 19327 19328 19329 19330 19331 19332 19333 19334 19335 19336 19337 19338 19339 19340 19341 19342 19343 19344 19345 19346 19347 19348 19349 19350 19351 19352 19353 19354 19355 19356 19357 19358 19359 19360 19361 19362 19363 19364 19365 19366 19367 19368 19369 19370 19371 19372 19373 19374 19375 19376 19377 19378 19379 19380 19381 19382 19383 19384 19385 19386 19387 19388 19389 19390 19391 19392 19393 19394 19395 19396 19397 19398 19399 19400 19401 19402 19403 19404 19405 19406 19407 19408 19409 19410 19411 19412 19413 19414 19415 19416 19417 19418 19419 19420 19421 19422 19423 19424 19425 19426 19427 19428 19429 19430 19431 19432 19433 19434 19435 19436 19437 19438 19439 19440 19441 19442 19443 19444 19445 19446 19447 19448 19449 19450 19451 19452 19453 19454 19455 19456 19457 19458 19459 19460 19461 19462 19463 19464 19465 19466 19467 19468 19469 19470 19471 19472 19473 19474 19475 19476 19477 19478 19479 19480 19481 19482 19483 19484 19485 19486 19487 19488 19489 19490 19491 19492 19493 19494 19495 19496 19497 19498 19499 19500 19501 19502 19503 19504 19505 19506 19507 19508 19509 19510 19511 19512 19513 19514 19515 19516 19517 19518 19519 19520 19521 19522 19523 19524 19525 19526 19527 19528 19529 19530 19531 19532 19533 19534 19535 19536 19537 19538 19539 19540 19541 19542 19543 19544 19545 19546 19547 19548 19549 19550 19551 19552 19553 19554 19555 19556 19557 19558 19559 19560 19561 19562 19563 19564 19565 19566 19567 19568 19569 19570 19571 19572 19573 19574 19575 19576 19577 19578 19579 19580 19581 19582 19583 19584 19585 19586 19587 19588 19589 19590 19591 19592 19593 19594 19595 19596 19597 19598 19599 19600 19601 19602 19603 19604 19605 19606 19607 19608 19609 19610 19611 19612 19613 19614 19615 19616 19617 19618 19619 19620 19621 19622 19623 19624 19625 19626 19627 19628 19629 19630 19631 19632 19633 19634 19635 19636 19637 19638 19639 19640 19641 19642 19643 19644 19645 19646 19647 19648 19649 19650 19651 19652 19653 19654 19655 19656 19657 19658 19659 19660 19661 19662 19663 19664 19665 19666 19667 19668 19669 19670 19671 19672 19673 19674 19675 19676 19677 19678 19679 19680 19681 19682 19683 19684 19685 19686 19687 19688 19689 19690 19691 19692 19693 19694 19695 19696 19697 19698 19699 19700 19701 19702 19703 19704 19705 19706 19707 19708 19709 19710 19711 19712 19713 19714 19715 19716 19717 19718 19719 19720 19721 19722 19723 19724 19725 19726 19727 19728 19729 19730 19731 19732 19733 19734 19735 19736 19737 19738 19739 19740 19741 19742 19743 19744 19745 19746 19747 19748 19749 19750 19751 19752 19753 19754 19755 19756 19757 19758 19759 19760 19761 19762 19763 19764 19765 19766 19767 19768 19769 19770 19771 19772 19773 19774 19775 19776 19777 19778 19779 19780 19781 19782 19783 19784 19785 19786 19787 19788 19789 19790 19791 19792 19793 19794 19795 19796 19797 19798 19799 19800 19801 19802 19803 19804 19805 19806 19807 19808 19809 19810 19811 19812 19813 19814 19815 19816 19817 19818 19819 19820 19821 19822 19823 19824 19825 19826 19827 19828 19829 19830 19831 19832 19833 19834 19835 19836 19837 19838 19839 19840 19841 19842 19843 19844 19845 19846 19847 19848 19849 19850 19851 19852 19853 19854 19855 19856 19857 19858 19859 19860 19861 19862 19863 19864 19865 19866 19867 19868 19869 19870 19871 19872 19873 19874 19875 19876 19877 19878 19879 19880 19881 19882 19883 19884 19885 19886 19887 19888 19889 19890 19891 19892 19893 19894 19895 19896 19897 19898 19899 19900 19901 19902 19903 19904 19905 19906 19907 19908 19909 19910 19911 19912 19913 19914 19915 19916 19917 19918 19919 19920 19921 19922 19923 19924 19925 19926 19927 19928 19929 19930 19931 19932 19933 19934 19935 19936 19937 19938 19939 19940 19941 19942 19943 19944 19945 19946 19947 19948 19949 19950 19951 19952 19953 19954 19955 19956 19957 19958 19959 19960 19961 19962 19963 19964 19965 19966 19967 19968 19969 19970 19971 19972 19973 19974 19975 19976 19977 19978 19979 19980 19981 19982 19983 19984 19985 19986 19987 19988 19989 19990 19991 19992 19993 19994 19995 19996 19997 19998 19999 20000 20001 20002 20003 20004 20005 20006 20007 20008 20009 20010 20011 20012 20013 20014 20015 20016 20017 20018 20019 20020 20021 20022 20023 20024 20025 20026 20027 20028 20029 20030 20031 20032 20033 20034 20035 20036 20037 20038 20039 20040 20041 20042 20043 20044 20045 20046 20047 20048 20049 20050 20051 20052 20053 20054 20055 20056 20057 20058 20059 20060 20061 20062 20063 20064 20065 20066 20067 20068 20069 20070 20071 20072 20073 20074 20075 20076 20077 20078 20079 20080 20081 20082 20083 20084 20085 20086 20087 20088 20089 20090 20091 20092 20093 20094 20095 20096 20097 20098 20099 20100 20101 20102 20103 20104 20105 20106 20107 20108 20109 20110 20111 20112 20113 20114 20115 20116 20117 20118 20119 20120 20121 20122 20123 20124 20125 20126 20127 20128 20129 20130 20131 20132 20133 20134 20135 20136 20137 20138 20139 20140 20141 20142 20143 20144 20145 20146 20147 20148 20149 20150 20151 20152 20153 20154 20155 20156 20157 20158 20159 20160 20161 20162 20163 20164 20165 20166 20167 20168 20169 20170 20171 20172 20173 20174 20175 20176 20177 20178 20179 20180 20181 20182 20183 20184 20185 20186 20187 20188 20189 20190 20191 20192 20193 20194 20195 20196 20197 20198 20199 20200 20201 20202 20203 20204 20205 20206 20207 20208 20209 20210 20211 20212 20213 20214 20215 20216 20217 20218 20219 20220 20221 20222 20223 20224 20225 20226 20227 20228 20229 20230 20231 20232 20233 20234 20235 20236 20237 20238 20239 20240 20241 20242 20243 20244 20245 20246 20247 20248 20249 20250 20251 20252 20253 20254 20255 20256 20257 20258 20259 20260 20261 20262 20263 20264 20265 20266 20267 20268 20269 20270 20271 20272 20273 20274 20275 20276 20277 20278 20279 20280 20281 20282 20283 20284 20285 20286 20287 20288 20289 20290 20291 20292 20293 20294 20295 20296 20297 20298 20299 20300 20301 20302 20303 20304 20305 20306 20307 20308 20309 20310 20311 20312 20313 20314 20315 20316 20317 20318 20319 20320 20321 20322 20323 20324 20325 20326 20327 20328 20329 20330 20331 20332 20333 20334 20335 20336 20337 20338 20339 20340 20341 20342 20343 20344 20345 20346 20347 20348 20349 20350 20351 20352 20353 20354 20355 20356 20357 20358 20359 20360 20361 20362 20363 20364 20365 20366 20367 20368 20369 20370 20371 20372 20373 20374 20375 20376 20377 20378 20379 20380 20381 20382 20383 20384 20385 20386 20387 20388 20389 20390 20391 20392 20393 20394 20395 20396 20397 20398 20399 20400 20401 20402 20403 20404 20405 20406 20407 20408 20409 20410 20411 20412 20413 20414 20415 20416 20417 20418 20419 20420 20421 20422 20423 20424 20425 20426 20427 20428 20429 20430 20431 20432 20433 20434 20435 20436 20437 20438 20439 20440 20441 20442 20443 20444 20445 20446 20447 20448 20449 20450 20451 20452 20453 20454 20455 20456 20457 20458 20459 20460 20461 20462 20463 20464 20465 20466 20467 20468 20469 20470 20471 20472 20473 20474 20475 20476 20477 20478 20479 20480 20481 20482 20483 20484 20485 20486 20487 20488 20489 20490 20491 20492 20493 20494 20495 20496 20497 20498 20499 20500 20501 20502 20503 20504 20505 20506 20507 20508 20509 20510 20511 20512 20513 20514 20515 20516 20517 20518 20519 20520 20521 20522 20523 20524 20525 20526 20527 20528 20529 20530 20531 20532 20533 20534 20535 20536 20537 20538 20539 20540 20541 20542 20543 20544 20545 20546 20547 20548 20549 20550 20551 20552 20553 20554 20555 20556 20557 20558 20559 20560 20561 20562 20563 20564 20565 20566 20567 20568 20569 20570 20571 20572 20573 20574 20575 20576 20577 20578 20579 20580 20581 20582 20583 20584 20585 20586 20587 20588 20589 20590 20591 20592 20593 20594 20595 20596 20597 20598 20599 20600 20601 20602 20603 20604 20605 20606 20607 20608 20609 20610 20611 20612 20613 20614 20615 20616 20617 20618 20619 20620 20621 20622 20623 20624 20625 20626 20627 20628 20629 20630 20631 20632 20633 20634 20635 20636 20637 20638 20639 20640 20641 20642 20643 20644 20645 20646 20647 20648 20649 20650 20651 20652 20653 20654 20655 20656 20657 20658 20659 20660 20661 20662 20663 20664 20665 20666 20667 20668 20669 20670 20671 20672 20673 20674 20675 20676 20677 20678 20679 20680 20681 20682 20683 20684 20685 20686 20687 20688 20689 20690 20691 20692 20693 20694 20695 20696 20697 20698 20699 20700 20701 20702 20703 20704 20705 20706 20707 20708 20709 20710 20711 20712 20713 20714 20715 20716 20717 20718 20719 20720 20721 20722 20723 20724 20725 20726 20727 20728 20729 20730 20731 20732 20733 20734 20735 20736 20737 20738 20739 20740 20741 20742 20743 20744 20745 20746 20747 20748 20749 20750 20751 20752 20753 20754 20755 20756 20757 20758 20759 20760 20761 20762 20763 20764 20765 20766 20767 20768 20769 20770 20771 20772 20773 20774 20775 20776 20777 20778 20779 20780 20781 20782 20783 20784 20785 20786 20787 20788 20789 20790 20791 20792 20793 20794 20795 20796 20797 20798 20799 20800 20801 20802 20803 20804 20805 20806 20807 20808 20809 20810 20811 20812 20813 20814 20815 20816 20817 20818 20819 20820 20821 20822 20823 20824 20825 20826 20827 20828 20829 20830 20831 20832 20833 20834 20835 20836 20837 20838 20839 20840 20841 20842 20843 20844 20845 20846 20847 20848 20849 20850 20851 20852 20853 20854 20855 20856 20857 20858 20859 20860 20861 20862 20863 20864 20865 20866 20867 20868 20869 20870 20871 20872 20873 20874 20875 20876 20877 20878 20879 20880 20881 20882 20883 20884 20885 20886 20887 20888 20889 20890 20891 20892 20893 20894 20895 20896 20897 20898 20899 20900 20901 20902 20903 20904 20905 20906 20907 20908 20909 20910 20911 20912 20913 20914 20915 20916 20917 20918 20919 20920 20921 20922 20923 20924 20925 20926 20927 20928 20929 20930 20931 20932 20933 20934 20935 20936 20937 20938 20939 20940 20941 20942 20943 20944 20945 20946 20947 20948 20949 20950 20951 20952 20953 20954 20955 20956 20957 20958 20959 20960 20961 20962 20963 20964 20965 20966 20967 20968 20969 20970 20971 20972 20973 20974 20975 20976 20977 20978 20979 20980 20981 20982 20983 20984 20985 20986 20987 20988 20989 20990 20991 20992 20993 20994 20995 20996 20997 20998 20999 21000 21001 21002 21003 21004 21005 21006 21007 21008 21009 21010 21011 21012 21013 21014 21015 21016 21017 21018 21019 21020 21021 21022 21023 21024 21025 21026 21027 21028 21029 21030 21031 21032 21033 21034 21035 21036 21037 21038 21039 21040 21041 21042 21043 21044 21045 21046 21047 21048 21049 21050 21051 21052 21053 21054 21055 21056 21057 21058 21059 21060 21061 21062 21063 21064 21065 21066 21067 21068 21069 21070 21071 21072 21073 21074 21075 21076 21077 21078 21079 21080 21081 21082 21083 21084 21085 21086 21087 21088 21089 21090 21091 21092 21093 21094 21095 21096 21097 21098 21099 21100 21101 21102 21103 21104 21105 21106 21107 21108 21109 21110 21111 21112 21113 21114 21115 21116 21117 21118 21119 21120 21121 21122 21123 21124 21125 21126 21127 21128 21129 21130 21131 21132 21133 21134 21135 21136 21137 21138 21139 21140 21141 21142 21143 21144 21145 21146 21147 21148 21149 21150 21151 21152 21153 21154 21155 21156 21157 21158 21159 21160 21161 21162 21163 21164 21165 21166 21167 21168 21169 21170 21171 21172 21173 21174 21175 21176 21177 21178 21179 21180 21181 21182 21183 21184 21185 21186 21187 21188 21189 21190 21191 21192 21193 21194 21195 21196 21197 21198 21199 21200 21201 21202 21203 21204 21205 21206 21207 21208 21209 21210 21211 21212 21213 21214 21215 21216 21217 21218 21219 21220 21221 21222 21223 21224 21225 21226 21227 21228 21229 21230 21231 21232 21233 21234 21235 21236 21237 21238 21239 21240 21241 21242 21243 21244 21245 21246 21247 21248 21249 21250 21251 21252 21253 21254 21255 21256 21257 21258 21259 21260 21261 21262 21263 21264 21265 21266 21267 21268 21269 21270 21271 21272 21273 21274 21275 21276 21277 21278 21279 21280 21281 21282 21283 21284 21285 21286 21287 21288 21289 21290 21291 21292 21293 21294 21295 21296 21297 21298 21299 21300 21301 21302 21303 21304 21305 21306 21307 21308 21309 21310 21311 21312 21313 21314 21315 21316 21317 21318 21319 21320 21321 21322 21323 21324 21325 21326 21327 21328 21329 21330 21331 21332 21333 21334 21335 21336 21337 21338 21339 21340 21341 21342 21343 21344 21345 21346 21347 21348 21349 21350 21351 21352 21353 21354 21355 21356 21357 21358 21359 21360 21361 21362 21363 21364 21365 21366 21367 21368 21369 21370 21371 21372 21373 21374 21375 21376 21377 21378 21379 21380 21381 21382 21383 21384 21385 21386 21387 21388 21389 21390 21391 21392 21393 21394 21395 21396 21397 21398 21399 21400 21401 21402 21403 21404 21405 21406 21407 21408 21409 21410 21411 21412 21413 21414 21415 21416 21417 21418 21419 21420 21421 21422 21423 21424 21425 21426 21427 21428 21429 21430 21431 21432 21433 21434 21435 21436 21437 21438 21439 21440 21441 21442 21443 21444 21445 21446 21447 21448 21449 21450 21451 21452 21453 21454 21455 21456 21457 21458 21459 21460 21461 21462 21463 21464 21465 21466 21467 21468 21469 21470 21471 21472 21473 21474 21475 21476 21477 21478 21479 21480 21481 21482 21483 21484 21485 21486 21487 21488 21489 21490 21491 21492 21493 21494 21495 21496 21497 21498 21499 21500 21501 21502 21503 21504 21505 21506 21507 21508 21509 21510 21511 21512 21513 21514 21515 21516 21517 21518 21519 21520 21521 21522 21523 21524 21525 21526 21527 21528 21529 21530 21531 21532 21533 21534 21535 21536 21537 21538 21539 21540 21541 21542 21543 21544 21545 21546 21547 21548 21549 21550 21551 21552 21553 21554 21555 21556 21557 21558 21559 21560 21561 21562 21563 21564 21565 21566 21567 21568 21569 21570 21571 21572 21573 21574 21575 21576 21577 21578 21579 21580 21581 21582 21583 21584 21585 21586 21587 21588 21589 21590 21591 21592 21593 21594 21595 21596 21597 21598 21599 21600 21601 21602 21603 21604 21605 21606 21607 21608 21609 21610 21611 21612 21613 21614 21615 21616 21617 21618 21619 21620 21621 21622 21623 21624 21625 21626 21627 21628 21629 21630 21631 21632 21633 21634 21635 21636 21637 21638 21639 21640 21641 21642 21643 21644 21645 21646 21647 21648 21649 21650 21651 21652 21653 21654 21655 21656 21657 21658 21659 21660 21661 21662 21663 21664 21665 21666 21667 21668 21669 21670 21671 21672 21673 21674 21675 21676 21677 21678 21679 21680 21681 21682 21683 21684 21685 21686 21687 21688 21689 21690 21691 21692 21693 21694 21695 21696 21697 21698 21699 21700 21701 21702 21703 21704 21705 21706 21707 21708 21709 21710 21711 21712 21713 21714 21715 21716 21717 21718 21719 21720 21721 21722 21723 21724 21725 21726 21727 21728 21729 21730 21731 21732 21733 21734 21735 21736 21737 21738 21739 21740 21741 21742 21743 21744 21745 21746 21747 21748 21749 21750 21751 21752 21753 21754 21755 21756 21757 21758 21759 21760 21761 21762 21763 21764 21765 21766 21767 21768 21769 21770 21771 21772 21773 21774 21775 21776 21777 21778 21779 21780 21781 21782 21783 21784 21785 21786 21787 21788 21789 21790 21791 21792 21793 21794 21795 21796 21797 21798 21799 21800 21801 21802 21803 21804 21805 21806 21807 21808 21809 21810 21811 21812 21813 21814 21815 21816 21817 21818 21819 21820 21821 21822 21823 21824 21825 21826 21827 21828 21829 21830 21831 21832 21833 21834 21835 21836 21837 21838 21839 21840 21841 21842 21843 21844 21845 21846 21847 21848 21849 21850 21851 21852 21853 21854 21855 21856 21857 21858 21859 21860 21861 21862 21863 21864 21865 21866 21867 21868 21869 21870 21871 21872 21873 21874 21875 21876 21877 21878 21879 21880 21881 21882 21883 21884 21885 21886 21887 21888 21889 21890 21891 21892 21893 21894 21895 21896 21897 21898 21899 21900 21901 21902 21903 21904 21905 21906 21907 21908 21909 21910 21911 21912 21913 21914 21915 21916 21917 21918 21919 21920 21921 21922 21923 21924 21925 21926 21927 21928 21929 21930 21931 21932 21933 21934 21935 21936 21937 21938 21939 21940 21941 21942 21943 21944 21945 21946 21947 21948 21949 21950 21951 21952 21953 21954 21955 21956 21957 21958 21959 21960 21961 21962 21963 21964 21965 21966 21967 21968 21969 21970 21971 21972 21973 21974 21975 21976 21977 21978 21979 21980 21981 21982 21983 21984 21985 21986 21987 21988 21989 21990 21991 21992 21993 21994 21995 21996 21997 21998 21999 22000 22001 22002 22003 22004 22005 22006 22007 22008 22009 22010 22011 22012 22013 22014 22015 22016 22017 22018 22019 22020 22021 22022 22023 22024 22025 22026 22027 22028 22029 22030 22031 22032 22033 22034 22035 22036 22037 22038 22039 22040 22041 22042 22043 22044 22045 22046 22047 22048 22049 22050 22051 22052 22053 22054 22055 22056 22057 22058 22059 22060 22061 22062 22063 22064 22065 22066 22067 22068 22069 22070 22071 22072 22073 22074 22075 22076 22077 22078 22079 22080 22081 22082 22083 22084 22085 22086 22087 22088 22089 22090 22091 22092 22093 22094 22095 22096 22097 22098 22099 22100 22101 22102 22103 22104 22105 22106 22107 22108 22109 22110 22111 22112 22113 22114 22115 22116 22117 22118 22119 22120 22121 22122 22123 22124 22125 22126 22127 22128 22129 22130 22131 22132 22133 22134 22135 22136 22137 22138 22139 22140 22141 22142 22143 22144 22145 22146 22147 22148 22149 22150 22151 22152 22153 22154 22155 22156 22157 22158 22159 22160 22161 22162 22163 22164 22165 22166 22167 22168 22169 22170 22171 22172 22173 22174 22175 22176 22177 22178 22179 22180 22181 22182 22183 22184 22185 22186 22187 22188 22189 22190 22191 22192 22193 22194 22195 22196 22197 22198 22199 22200 22201 22202 22203 22204 22205 22206 22207 22208 22209 22210 22211 22212 22213 22214 22215 22216 22217 22218 22219 22220 22221 22222 22223 22224 22225 22226 22227 22228 22229 22230 22231 22232 22233 22234 22235 22236 22237 22238 22239 22240 22241 22242 22243 22244 22245 22246 22247 22248 22249 22250 22251 22252 22253 22254 22255 22256 22257 22258 22259 22260 22261 22262 22263 22264 22265 22266 22267 22268 22269 22270 22271 22272 22273 22274 22275 22276 22277 22278 22279 22280 22281 22282 22283 22284 22285 22286 22287 22288 22289 22290 22291 22292 22293 22294 22295 22296 22297 22298 22299 22300 22301 22302 22303 22304 22305 22306 22307 22308 22309 22310 22311 22312 22313 22314 22315 22316 22317 22318 22319 22320 22321 22322 22323 22324 22325 22326 22327 22328 22329 22330 22331 22332 22333 22334 22335 22336 22337 22338 22339 22340 22341 22342 22343 22344 22345 22346 22347 22348 22349 22350 22351 22352 22353 22354 22355 22356 22357 22358 22359 22360 22361 22362 22363 22364 22365 22366 22367 22368 22369 22370 22371 22372 22373 22374 22375 22376 22377 22378 22379 22380 22381 22382 22383 22384 22385 22386 22387 22388 22389 22390 22391 22392 22393 22394 22395 22396 22397 22398 22399 22400 22401 22402 22403 22404 22405 22406 22407 22408 22409 22410 22411 22412 22413 22414 22415 22416 22417 22418 22419 22420 22421 22422 22423 22424 22425 22426 22427 22428 22429 22430 22431 22432 22433 22434 22435 22436 22437 22438 22439 22440 22441 22442 22443 22444 22445 22446 22447 22448 22449 22450 22451 22452 22453 22454 22455 22456 22457 22458 22459 22460 22461 22462 22463 22464 22465 22466 22467 22468 22469 22470 22471 22472 22473 22474 22475 22476 22477 22478 22479 22480 22481 22482 22483 22484 22485 22486 22487 22488 22489 22490 22491 22492 22493 22494 22495 22496 22497 22498 22499 22500 22501 22502 22503 22504 22505 22506 22507 22508 22509 22510 22511 22512 22513 22514 22515 22516 22517 22518 22519 22520 22521 22522 22523 22524 22525 22526 22527 22528 22529 22530 22531 22532 22533 22534 22535 22536 22537 22538 22539 22540 22541 22542 22543 22544 22545 22546 22547 22548 22549 22550 22551 22552 22553 22554 22555 22556 22557 22558 22559 22560 22561 22562 22563 22564 22565 22566 22567 22568 22569 22570 22571 22572 22573 22574 22575 22576 22577 22578 22579 22580 22581 22582 22583 22584 22585 22586 22587 22588 22589 22590 22591 22592 22593 22594 22595 22596 22597 22598 22599 22600 22601 22602 22603 22604 22605 22606 22607 22608 22609 22610 22611 22612 22613 22614 22615 22616 22617 22618 22619 22620 22621 22622 22623 22624 22625 22626 22627 22628 22629 22630 22631 22632 22633 22634 22635 22636 22637 22638 22639 22640 22641 22642 22643 22644 22645 22646 22647 22648 22649 22650 22651 22652 22653 22654 22655 22656 22657 22658 22659 22660 22661 22662 22663 22664 22665 22666 22667 22668 22669 22670 22671 22672 22673 22674 22675 22676 22677 22678 22679 22680 22681 22682 22683 22684 22685 22686 22687 22688 22689 22690 22691 22692 22693 22694 22695 22696 22697 22698 22699 22700 22701 22702 22703 22704 22705 22706 22707 22708 22709 22710 22711 22712 22713 22714 22715 22716 22717 22718 22719 22720 22721 22722 22723 22724 22725 22726 22727 22728 22729 22730 22731 22732 22733 22734 22735 22736 22737 22738 22739 22740 22741 22742 22743 22744 22745 22746 22747 22748 22749 22750 22751 22752 22753 22754 22755 22756 22757 22758 22759 22760 22761 22762 22763 22764 22765 22766 22767 22768 22769 22770 22771 22772 22773 22774 22775 22776 22777 22778 22779 22780 22781 22782 22783 22784 22785 22786 22787 22788 22789 22790 22791 22792 22793 22794 22795 22796 22797 22798 22799 22800 22801 22802 22803 22804 22805 22806 22807 22808 22809 22810 22811 22812 22813 22814 22815 22816 22817 22818 22819 22820 22821 22822 22823 22824 22825 22826 22827 22828 22829 22830 22831 22832 22833 22834 22835 22836 22837 22838 22839 22840 22841 22842 22843 22844 22845 22846 22847 22848 22849 22850 22851 22852 22853 22854 22855 22856 22857 22858 22859 22860 22861 22862 22863 22864 22865 22866 22867 22868 22869 22870 22871 22872 22873 22874 22875 22876 22877 22878 22879 22880 22881 22882 22883 22884 22885 22886 22887 22888 22889 22890 22891 22892 22893 22894 22895 22896 22897 22898 22899 22900 22901 22902 22903 22904 22905 22906 22907 22908 22909 22910 22911 22912 22913 22914 22915 22916 22917 22918 22919 22920 22921 22922 22923 22924 22925 22926 22927 22928 22929 22930 22931 22932 22933 22934 22935 22936 22937 22938 22939 22940 22941 22942 22943 22944 22945 22946 22947 22948 22949 22950 22951 22952 22953 22954 22955 22956 22957 22958 22959 22960 22961 22962 22963 22964 22965 22966 22967 22968 22969 22970 22971 22972 22973 22974 22975 22976 22977 22978 22979 22980 22981 22982 22983 22984 22985 22986 22987 22988 22989 22990 22991 22992 22993 22994 22995 22996 22997 22998 22999 23000 23001 23002 23003 23004 23005 23006 23007 23008 23009 23010 23011 23012 23013 23014 23015 23016 23017 23018 23019 23020 23021 23022 23023 23024 23025 23026 23027 23028 23029 23030 23031 23032 23033 23034 23035 23036 23037 23038 23039 23040 23041 23042 23043 23044 23045 23046 23047 23048 23049 23050 23051 23052 23053 23054 23055 23056 23057 23058 23059 23060 23061 23062 23063 23064 23065 23066 23067 23068 23069 23070 23071 23072 23073 23074 23075 23076 23077 23078 23079 23080 23081 23082 23083 23084 23085 23086 23087 23088 23089 23090 23091 23092 23093 23094 23095 23096 23097 23098 23099 23100 23101 23102 23103 23104 23105 23106 23107 23108 23109 23110 23111 23112 23113 23114 23115 23116 23117 23118 23119 23120 23121 23122 23123 23124 23125 23126 23127 23128 23129 23130 23131 23132 23133 23134 23135 23136 23137 23138 23139 23140 23141 23142 23143 23144 23145 23146 23147 23148 23149 23150 23151 23152 23153 23154 23155 23156 23157 23158 23159 23160 23161 23162 23163 23164 23165 23166 23167 23168 23169 23170 23171 23172 23173 23174 23175 23176 23177 23178 23179 23180 23181 23182 23183 23184 23185 23186 23187 23188 23189 23190 23191 23192 23193 23194 23195 23196 23197 23198 23199 23200 23201 23202 23203 23204 23205 23206 23207 23208 23209 23210 23211 23212 23213 23214 23215 23216 23217 23218 23219 23220 23221 23222 23223 23224 23225 23226 23227 23228 23229 23230 23231 23232 23233 23234 23235 23236 23237 23238 23239 23240 23241 23242 23243 23244 23245 23246 23247 23248 23249 23250 23251 23252 23253 23254 23255 23256 23257 23258 23259 23260 23261 23262 23263 23264 23265 23266 23267 23268 23269 23270 23271 23272 23273 23274 23275 23276 23277 23278 23279 23280 23281 23282 23283 23284 23285 23286 23287 23288 23289 23290 23291 23292 23293 23294 23295 23296 23297 23298 23299 23300 23301 23302 23303 23304 23305 23306 23307 23308 23309 23310 23311 23312 23313 23314 23315 23316 23317 23318 23319 23320 23321 23322 23323 23324 23325 23326 23327 23328 23329 23330 23331 23332 23333 23334 23335 23336 23337 23338 23339 23340 23341 23342 23343 23344 23345 23346 23347 23348 23349 23350 23351 23352 23353 23354 23355 23356 23357 23358 23359 23360 23361 23362 23363 23364 23365 23366 23367 23368 23369 23370 23371 23372 23373 23374 23375 23376 23377 23378 23379 23380 23381 23382 23383 23384 23385 23386 23387 23388 23389 23390 23391 23392 23393 23394 23395 23396 23397 23398 23399 23400 23401 23402 23403 23404 23405 23406 23407 23408 23409 23410 23411 23412 23413 23414 23415 23416 23417 23418 23419 23420 23421 23422 23423 23424 23425 23426 23427 23428 23429 23430 23431 23432 23433 23434 23435 23436 23437 23438 23439 23440 23441 23442 23443 23444 23445 23446 23447 23448 23449 23450 23451 23452 23453 23454 23455 23456 23457 23458 23459 23460 23461 23462 23463 23464 23465 23466 23467 23468 23469 23470 23471 23472 23473 23474 23475 23476 23477 23478 23479 23480 23481 23482 23483 23484 23485 23486 23487 23488 23489 23490 23491 23492 23493 23494 23495 23496 23497 23498 23499 23500 23501 23502 23503 23504 23505 23506 23507 23508 23509 23510 23511 23512 23513 23514 23515 23516 23517 23518 23519 23520 23521 23522 23523 23524 23525 23526 23527 23528 23529 23530 23531 23532 23533 23534 23535 23536 23537 23538 23539 23540 23541 23542 23543 23544 23545 23546 23547 23548 23549 23550 23551 23552 23553 23554 23555 23556 23557 23558 23559 23560 23561 23562 23563 23564 23565 23566 23567 23568 23569 23570 23571 23572 23573 23574 23575 23576 23577 23578 23579 23580 23581 23582 23583 23584 23585 23586 23587 23588 23589 23590 23591 23592 23593 23594 23595 23596 23597 23598 23599 23600 23601 23602 23603 23604 23605 23606 23607 23608 23609 23610 23611 23612 23613 23614 23615 23616 23617 23618 23619 23620 23621 23622 23623 23624 23625 23626 23627 23628 23629 23630 23631 23632 23633 23634 23635 23636 23637 23638 23639 23640 23641 23642 23643 23644 23645 23646 23647 23648 23649 23650 23651 23652 23653 23654 23655 23656 23657 23658 23659 23660 23661 23662 23663 23664 23665 23666 23667 23668 23669 23670 23671 23672 23673 23674 23675 23676 23677 23678 23679 23680 23681 23682 23683 23684 23685 23686 23687 23688 23689 23690 23691 23692 23693 23694 23695 23696 23697 23698 23699 23700 23701 23702 23703 23704 23705 23706 23707 23708 23709 23710 23711 23712 23713 23714 23715 23716 23717 23718 23719 23720 23721 23722 23723 23724 23725 23726 23727 23728 23729 23730 23731 23732 23733 23734 23735 23736 23737 23738 23739 23740 23741 23742 23743 23744 23745 23746 23747 23748 23749 23750 23751 23752 23753 23754 23755 23756 23757 23758 23759 23760 23761 23762 23763 23764 23765 23766 23767 23768 23769 23770 23771 23772 23773 23774 23775 23776 23777 23778 23779 23780 23781 23782 23783 23784 23785 23786 23787 23788 23789 23790 23791 23792 23793 23794 23795 23796 23797 23798 23799 23800 23801 23802 23803 23804 23805 23806 23807 23808 23809 23810 23811 23812 23813 23814 23815 23816 23817 23818 23819 23820 23821 23822 23823 23824 23825 23826 23827 23828 23829 23830 23831 23832 23833 23834 23835 23836 23837 23838 23839 23840 23841 23842 23843 23844 23845 23846 23847 23848 23849 23850 23851 23852 23853 23854 23855 23856 23857 23858 23859 23860 23861 23862 23863 23864 23865 23866 23867 23868 23869 23870 23871 23872 23873 23874 23875 23876 23877 23878 23879 23880 23881 23882 23883 23884 23885 23886 23887 23888 23889 23890 23891 23892 23893 23894 23895 23896 23897 23898 23899 23900 23901 23902 23903 23904 23905 23906 23907 23908 23909 23910 23911 23912 23913 23914 23915 23916 23917 23918 23919 23920 23921 23922 23923 23924 23925 23926 23927 23928 23929 23930 23931 23932 23933 23934 23935 23936 23937 23938 23939 23940 23941 23942 23943 23944 23945 23946 23947 23948 23949 23950 23951 23952 23953 23954 23955 23956 23957 23958 23959 23960 23961 23962 23963 23964 23965 23966 23967 23968 23969 23970 23971 23972 23973 23974 23975 23976 23977 23978 23979 23980 23981 23982 23983 23984 23985 23986 23987 23988 23989 23990 23991 23992 23993 23994 23995 23996 23997 23998 23999 24000 24001 24002 24003 24004 24005 24006 24007 24008 24009 24010 24011 24012 24013 24014 24015 24016 24017 24018 24019 24020 24021 24022 24023 24024 24025 24026 24027 24028 24029 24030 24031 24032 24033 24034 24035 24036 24037 24038 24039 24040 24041 24042 24043 24044 24045 24046 24047 24048 24049 24050 24051 24052 24053 24054 24055 24056 24057 24058 24059 24060 24061 24062 24063 24064 24065 24066 24067 24068 24069 24070 24071 24072 24073 24074 24075 24076 24077 24078 24079 24080 24081 24082 24083 24084 24085 24086 24087 24088 24089 24090 24091 24092 24093 24094 24095 24096 24097 24098 24099 24100 24101 24102 24103 24104 24105 24106 24107 24108 24109 24110 24111 24112 24113 24114 24115 24116 24117 24118 24119 24120 24121 24122 24123 24124 24125 24126 24127 24128 24129 24130 24131 24132 24133 24134 24135 24136 24137 24138 24139 24140 24141 24142 24143 24144 24145 24146 24147 24148 24149 24150 24151 24152 24153 24154 24155 24156 24157 24158 24159 24160 24161 24162 24163 24164 24165 24166 24167 24168 24169 24170 24171 24172 24173 24174 24175 24176 24177 24178 24179 24180 24181 24182 24183 24184 24185 24186 24187 24188 24189 24190 24191 24192 24193 24194 24195 24196 24197 24198 24199 24200 24201 24202 24203 24204 24205 24206 24207 24208 24209 24210 24211 24212 24213 24214 24215 24216 24217 24218 24219 24220 24221 24222 24223 24224 24225 24226 24227 24228 24229 24230 24231 24232 24233 24234 24235 24236 24237 24238 24239 24240 24241 24242 24243 24244 24245 24246 24247 24248 24249 24250 24251 24252 24253 24254 24255 24256 24257 24258 24259 24260 24261 24262 24263 24264 24265 24266 24267 24268 24269 24270 24271 24272 24273 24274 24275 24276 24277 24278 24279 24280 24281 24282 24283 24284 24285 24286 24287 24288 24289 24290 24291 24292 24293 24294 24295 24296 24297 24298 24299 24300 24301 24302 24303 24304 24305 24306 24307 24308 24309 24310 24311 24312 24313 24314 24315 24316 24317 24318 24319 24320 24321 24322 24323 24324 24325 24326 24327 24328 24329 24330 24331 24332 24333 24334 24335 24336 24337 24338 24339 24340 24341 24342 24343 24344 24345 24346 24347 24348 24349 24350 24351 24352 24353 24354 24355 24356 24357 24358 24359 24360 24361 24362 24363 24364 24365 24366 24367 24368 24369 24370 24371 24372 24373 24374 24375 24376 24377 24378 24379 24380 24381 24382 24383 24384 24385 24386 24387 24388 24389 24390 24391 24392 24393 24394 24395 24396 24397 24398 24399 24400 24401 24402 24403 24404 24405 24406 24407 24408 24409 24410 24411 24412 24413 24414 24415 24416 24417 24418 24419 24420 24421 24422 24423 24424 24425 24426 24427 24428 24429 24430 24431 24432 24433 24434 24435 24436 24437 24438 24439 24440 24441 24442 24443 24444 24445 24446 24447 24448 24449 24450 24451 24452 24453 24454 24455 24456 24457 24458 24459 24460 24461 24462 24463 24464 24465 24466 24467 24468 24469 24470 24471 24472 24473 24474 24475 24476 24477 24478 24479 24480 24481 24482 24483 24484 24485 24486 24487 24488 24489 24490 24491 24492 24493 24494 24495 24496 24497 24498 24499 24500 24501 24502 24503 24504 24505 24506 24507 24508 24509 24510 24511 24512 24513 24514 24515 24516 24517 24518 24519 24520 24521 24522 24523 24524 24525 24526 24527 24528 24529 24530 24531 24532 24533 24534 24535 24536 24537 24538 24539 24540 24541 24542 24543 24544 24545 24546 24547 24548 24549 24550 24551 24552 24553 24554 24555 24556 24557 24558 24559 24560 24561 24562 24563 24564 24565 24566 24567 24568 24569 24570 24571 24572 24573 24574 24575 24576 24577 24578 24579 24580 24581 24582 24583 24584 24585 24586 24587 24588 24589 24590 24591 24592 24593 24594 24595 24596 24597 24598 24599 24600 24601 24602 24603 24604 24605 24606 24607 24608 24609 24610 24611 24612 24613 24614 24615 24616 24617 24618 24619 24620 24621 24622 24623 24624 24625 24626 24627 24628 24629 24630 24631 24632 24633 24634 24635 24636 24637 24638 24639 24640 24641 24642 24643 24644 24645 24646 24647 24648 24649 24650 24651 24652 24653 24654 24655 24656 24657 24658 24659 24660 24661 24662 24663 24664 24665 24666 24667 24668 24669 24670 24671 24672 24673 24674 24675 24676 24677 24678 24679 24680 24681 24682 24683 24684 24685 24686 24687 24688 24689 24690 24691 24692 24693 24694 24695 24696 24697 24698 24699 24700 24701 24702 24703 24704 24705 24706 24707 24708 24709 24710 24711 24712 24713 24714 24715 24716 24717 24718 24719 24720 24721 24722 24723 24724 24725 24726 24727 24728 24729 24730 24731 24732 24733 24734 24735 24736 24737 24738 24739 24740 24741 24742 24743 24744 24745 24746 24747 24748 24749 24750 24751 24752 24753 24754 24755 24756 24757 24758 24759 24760 24761 24762 24763 24764 24765 24766 24767 24768 24769 24770 24771 24772 24773 24774 24775 24776 24777 24778 24779 24780 24781 24782 24783 24784 24785 24786 24787 24788 24789 24790 24791 24792 24793 24794 24795 24796 24797 24798 24799 24800 24801 24802 24803 24804 24805 24806 24807 24808 24809 24810 24811 24812 24813 24814 24815 24816 24817 24818 24819 24820 24821 24822 24823 24824 24825 24826 24827 24828 24829 24830 24831 24832 24833 24834 24835 24836 24837 24838 24839 24840 24841 24842 24843 24844 24845 24846 24847 24848 24849 24850 24851 24852 24853 24854 24855 24856 24857 24858 24859 24860 24861 24862 24863 24864 24865 24866 24867 24868 24869 24870 24871 24872 24873 24874 24875 24876 24877 24878 24879 24880 24881 24882 24883 24884 24885 24886 24887 24888 24889 24890 24891 24892 24893 24894 24895 24896 24897 24898 24899 24900 24901 24902 24903 24904 24905 24906 24907 24908 24909 24910 24911 24912 24913 24914 24915 24916 24917 24918 24919 24920 24921 24922 24923 24924 24925 24926 24927 24928 24929 24930 24931 24932 24933 24934 24935 24936 24937 24938 24939 24940 24941 24942 24943 24944 24945 24946 24947 24948 24949 24950 24951 24952 24953 24954 24955 24956 24957 24958 24959 24960 24961 24962 24963 24964 24965 24966 24967 24968 24969 24970 24971 24972 24973 24974 24975 24976 24977 24978 24979 24980 24981 24982 24983 24984 24985 24986 24987 24988 24989 24990 24991 24992 24993 24994 24995 24996 24997 24998 24999 25000 25001 25002 25003 25004 25005 25006 25007 25008 25009 25010 25011 25012 25013 25014 25015 25016 25017 25018 25019 25020 25021 25022 25023 25024 25025 25026 25027 25028 25029 25030 25031 25032 25033 25034 25035 25036 25037 25038 25039 25040 25041 25042 25043 25044 25045 25046 25047 25048 25049 25050 25051 25052 25053 25054 25055 25056 25057 25058 25059 25060 25061 25062 25063 25064 25065 25066 25067 25068 25069 25070 25071 25072 25073 25074 25075 25076 25077 25078 25079 25080 25081 25082 25083 25084 25085 25086 25087 25088 25089 25090 25091 25092 25093 25094 25095 25096 25097 25098 25099 25100 25101 25102 25103 25104 25105 25106 25107 25108 25109 25110 25111 25112 25113 25114 25115 25116 25117 25118 25119 25120 25121 25122 25123 25124 25125 25126 25127 25128 25129 25130 25131 25132 25133 25134 25135 25136 25137 25138 25139 25140 25141 25142 25143 25144 25145 25146 25147 25148 25149 25150 25151 25152 25153 25154 25155 25156 25157 25158 25159 25160 25161 25162 25163 25164 25165 25166 25167 25168 25169 25170 25171 25172 25173 25174 25175 25176 25177 25178 25179 25180 25181 25182 25183 25184 25185 25186 25187 25188 25189 25190 25191 25192 25193 25194 25195 25196 25197 25198 25199 25200 25201 25202 25203 25204 25205 25206 25207 25208 25209 25210 25211 25212 25213 25214 25215 25216 25217 25218 25219 25220 25221 25222 25223 25224 25225 25226 25227 25228 25229 25230 25231 25232 25233 25234 25235 25236 25237 25238 25239 25240 25241 25242 25243 25244 25245 25246 25247 25248 25249 25250 25251 25252 25253 25254 25255 25256 25257 25258 25259 25260 25261 25262 25263 25264 25265 25266 25267 25268 25269 25270 25271 25272 25273 25274 25275 25276 25277 25278 25279 25280 25281 25282 25283 25284 25285 25286 25287 25288 25289 25290 25291 25292 25293 25294 25295 25296 25297 25298 25299 25300 25301 25302 25303 25304 25305 25306 25307 25308 25309 25310 25311 25312 25313 25314 25315 25316 25317 25318 25319 25320 25321 25322 25323 25324 25325 25326 25327 25328 25329 25330 25331 25332 25333 25334 25335 25336 25337 25338 25339 25340 25341 25342 25343 25344 25345 25346 25347 25348 25349 25350 25351 25352 25353 25354 25355 25356 25357 25358 25359 25360 25361 25362 25363 25364 25365 25366 25367 25368 25369 25370 25371 25372 25373 25374 25375 25376 25377 25378 25379 25380 25381 25382 25383 25384 25385 25386 25387 25388 25389 25390 25391 25392 25393 25394 25395 25396 25397 25398 25399 25400 25401 25402 25403 25404 25405 25406 25407 25408 25409 25410 25411 25412 25413 25414 25415 25416 25417 25418 25419 25420 25421 25422 25423 25424 25425 25426 25427 25428 25429 25430 25431 25432 25433 25434 25435 25436 25437 25438 25439 25440 25441 25442 25443 25444 25445 25446 25447 25448 25449 25450 25451 25452 25453 25454 25455 25456 25457 25458 25459 25460 25461 25462 25463 25464 25465 25466 25467 25468 25469 25470 25471 25472 25473 25474 25475 25476 25477 25478 25479 25480 25481 25482 25483 25484 25485 25486 25487 25488 25489 25490 25491 25492 25493 25494 25495 25496 25497 25498 25499 25500 25501 25502 25503 25504 25505 25506 25507 25508 25509 25510 25511 25512 25513 25514 25515 25516 25517 25518 25519 25520 25521 25522 25523 25524 25525 25526 25527 25528 25529 25530 25531 25532 25533 25534 25535 25536 25537 25538 25539 25540 25541 25542 25543 25544 25545 25546 25547 25548 25549 25550 25551 25552 25553 25554 25555 25556 25557 25558 25559 25560 25561 25562 25563 25564 25565 25566 25567 25568 25569 25570 25571 25572 25573 25574 25575 25576 25577 25578 25579 25580 25581 25582 25583 25584 25585 25586 25587 25588 25589 25590 25591 25592 25593 25594 25595 25596 25597 25598 25599 25600 25601 25602 25603 25604 25605 25606 25607 25608 25609 25610 25611 25612 25613 25614 25615 25616 25617 25618 25619 25620 25621 25622 25623 25624 25625 25626 25627 25628 25629 25630 25631 25632 25633 25634 25635 25636 25637 25638 25639 25640 25641 25642 25643 25644 25645 25646 25647 25648 25649 25650 25651 25652 25653 25654 25655 25656 25657 25658 25659 25660 25661 25662 25663 25664 25665 25666 25667 25668 25669 25670 25671 25672 25673 25674 25675 25676 25677 25678 25679 25680 25681 25682 25683 25684 25685 25686 25687 25688 25689 25690 25691 25692 25693 25694 25695 25696 25697 25698 25699 25700 25701 25702 25703 25704 25705 25706 25707 25708 25709 25710 25711 25712 25713 25714 25715 25716 25717 25718 25719 25720 25721 25722 25723 25724 25725 25726 25727 25728 25729 25730 25731 25732 25733 25734 25735 25736 25737 25738 25739 25740 25741 25742 25743 25744 25745 25746 25747 25748 25749 25750 25751 25752 25753 25754 25755 25756 25757 25758 25759 25760 25761 25762 25763 25764 25765 25766 25767 25768 25769 25770 25771 25772 25773 25774 25775 25776 25777 25778 25779 25780 25781 25782 25783 25784 25785 25786 25787 25788 25789 25790 25791 25792 25793 25794 25795 25796 25797 25798 25799 25800 25801 25802 25803 25804 25805 25806 25807 25808 25809 25810 25811 25812 25813 25814 25815 25816 25817 25818 25819 25820 25821 25822 25823 25824 25825 25826 25827 25828 25829 25830 25831 25832 25833 25834 25835 25836 25837 25838 25839 25840 25841 25842 25843 25844 25845 25846 25847 25848 25849 25850 25851 25852 25853 25854 25855 25856 25857 25858 25859 25860 25861 25862 25863 25864 25865 25866 25867 25868 25869 25870 25871 25872 25873 25874 25875 25876 25877 25878 25879 25880 25881 25882 25883 25884 25885 25886 25887 25888 25889 25890 25891 25892 25893 25894 25895 25896 25897 25898 25899 25900 25901 25902 25903 25904 25905 25906 25907 25908 25909 25910 25911 25912 25913 25914 25915 25916 25917 25918 25919 25920 25921 25922 25923 25924 25925 25926 25927 25928 25929 25930 25931 25932 25933 25934 25935 25936 25937 25938 25939 25940 25941 25942 25943 25944 25945 25946 25947 25948 25949 25950 25951 25952 25953 25954 25955 25956 25957 25958 25959 25960 25961 25962 25963 25964 25965 25966 25967 25968 25969 25970 25971 25972 25973 25974 25975 25976 25977 25978 25979 25980 25981 25982 25983 25984 25985 25986 25987 25988 25989 25990 25991 25992 25993 25994 25995 25996 25997 25998 25999 26000 26001 26002 26003 26004 26005 26006 26007 26008 26009 26010 26011 26012 26013 26014 26015 26016 26017 26018 26019 26020 26021 26022 26023 26024 26025 26026 26027 26028 26029 26030 26031 26032 26033 26034 26035 26036 26037 26038 26039 26040 26041 26042 26043 26044 26045 26046 26047 26048 26049 26050 26051 26052 26053 26054 26055 26056 26057 26058 26059 26060 26061 26062 26063 26064 26065 26066 26067 26068 26069 26070 26071 26072 26073 26074 26075 26076 26077 26078 26079 26080 26081 26082 26083 26084 26085 26086 26087 26088 26089 26090 26091 26092 26093 26094 26095 26096 26097 26098 26099 26100 26101 26102 26103 26104 26105 26106 26107 26108 26109 26110 26111 26112 26113 26114 26115 26116 26117 26118 26119 26120 26121 26122 26123 26124 26125 26126 26127 26128 26129 26130 26131 26132 26133 26134 26135 26136 26137 26138 26139 26140 26141 26142 26143 26144 26145 26146 26147 26148 26149 26150 26151 26152 26153 26154 26155 26156 26157 26158 26159 26160 26161 26162 26163 26164 26165 26166 26167 26168 26169 26170 26171 26172 26173 26174 26175 26176 26177 26178 26179 26180 26181 26182 26183 26184 26185 26186 26187 26188 26189 26190 26191 26192 26193 26194 26195 26196 26197 26198 26199 26200 26201 26202 26203 26204 26205 26206 26207 26208 26209 26210 26211 26212 26213 26214 26215 26216 26217 26218 26219 26220 26221 26222 26223 26224 26225 26226 26227 26228 26229 26230 26231 26232 26233 26234 26235 26236 26237 26238 26239 26240 26241 26242 26243 26244 26245 26246 26247 26248 26249 26250 26251 26252 26253 26254 26255 26256 26257 26258 26259 26260 26261 26262 26263 26264 26265 26266 26267 26268 26269 26270 26271 26272 26273 26274 26275 26276 26277 26278 26279 26280 26281 26282 26283 26284 26285 26286 26287 26288 26289 26290 26291 26292 26293 26294 26295 26296 26297 26298 26299 26300 26301 26302 26303 26304 26305 26306 26307 26308 26309 26310 26311 26312 26313 26314 26315 26316 26317 26318 26319 26320 26321 26322 26323 26324 26325 26326 26327 26328 26329 26330 26331 26332 26333 26334 26335 26336 26337 26338 26339 26340 26341 26342 26343 26344 26345 26346 26347 26348 26349 26350 26351 26352 26353 26354 26355 26356 26357 26358 26359 26360 26361 26362 26363 26364 26365 26366 26367 26368 26369 26370 26371 26372 26373 26374 26375 26376 26377 26378 26379 26380 26381 26382 26383 26384 26385 26386 26387 26388 26389 26390 26391 26392 26393 26394 26395 26396 26397 26398 26399 26400 26401 26402 26403 26404 26405 26406 26407 26408 26409 26410 26411 26412 26413 26414 26415 26416 26417 26418 26419 26420 26421 26422 26423 26424 26425 26426 26427 26428 26429 26430 26431 26432 26433 26434 26435 26436 26437 26438 26439 26440 26441 26442 26443 26444 26445 26446 26447 26448 26449 26450 26451 26452 26453 26454 26455 26456 26457 26458 26459 26460 26461 26462 26463 26464 26465 26466 26467 26468 26469 26470 26471 26472 26473 26474 26475 26476 26477 26478 26479 26480 26481 26482 26483 26484 26485 26486 26487 26488 26489 26490 26491 26492 26493 26494 26495 26496 26497 26498 26499 26500 26501 26502 26503 26504 26505 26506 26507 26508 26509 26510 26511 26512 26513 26514 26515 26516 26517 26518 26519 26520 26521 26522 26523 26524 26525 26526 26527 26528 26529 26530 26531 26532 26533 26534 26535 26536 26537 26538 26539 26540 26541 26542 26543 26544 26545 26546 26547 26548 26549 26550 26551 26552 26553 26554 26555 26556 26557 26558 26559 26560 26561 26562 26563 26564 26565 26566 26567 26568 26569 26570 26571 26572 26573 26574 26575 26576 26577 26578 26579 26580 26581 26582 26583 26584 26585 26586 26587 26588 26589 26590 26591 26592 26593 26594 26595 26596 26597 26598 26599 26600 26601 26602 26603 26604 26605 26606 26607 26608 26609 26610 26611 26612 26613 26614 26615 26616 26617 26618 26619 26620 26621 26622 26623 26624 26625 26626 26627 26628 26629 26630 26631 26632 26633 26634 26635 26636 26637 26638 26639 26640 26641 26642 26643 26644 26645 26646 26647 26648 26649 26650 26651 26652 26653 26654 26655 26656 26657 26658 26659 26660 26661 26662 26663 26664 26665 26666 26667 26668 26669 26670 26671 26672 26673 26674 26675 26676 26677 26678 26679 26680 26681 26682 26683 26684 26685 26686 26687 26688 26689 26690 26691 26692 26693 26694 26695 26696 26697 26698 26699 26700 26701 26702 26703 26704 26705 26706 26707 26708 26709 26710 26711 26712 26713 26714 26715 26716 26717 26718 26719 26720 26721 26722 26723 26724 26725 26726 26727 26728 26729 26730 26731 26732 26733 26734 26735 26736 26737 26738 26739 26740 26741 26742 26743 26744 26745 26746 26747 26748 26749 26750 26751 26752 26753 26754 26755 26756 26757 26758 26759 26760 26761 26762 26763 26764 26765 26766 26767 26768 26769 26770 26771 26772 26773 26774 26775 26776 26777 26778 26779 26780 26781 26782 26783 26784 26785 26786 26787 26788 26789 26790 26791 26792 26793 26794 26795 26796 26797 26798 26799 26800 26801 26802 26803 26804 26805 26806 26807 26808 26809 26810 26811 26812 26813 26814 26815 26816 26817 26818 26819 26820 26821 26822 26823 26824 26825 26826 26827 26828 26829 26830 26831 26832 26833 26834 26835 26836 26837 26838 26839 26840 26841 26842 26843 26844 26845 26846 26847 26848 26849 26850 26851 26852 26853 26854 26855 26856 26857 26858 26859 26860 26861 26862 26863 26864 26865 26866 26867 26868 26869 26870 26871 26872 26873 26874 26875 26876 26877 26878 26879 26880 26881 26882 26883 26884 26885 26886 26887 26888 26889 26890 26891 26892 26893 26894 26895 26896 26897 26898 26899 26900 26901 26902 26903 26904 26905 26906 26907 26908 26909 26910 26911 26912 26913 26914 26915 26916 26917 26918 26919 26920 26921 26922 26923 26924 26925 26926 26927 26928 26929 26930 26931 26932 26933 26934 26935 26936 26937 26938 26939 26940 26941 26942 26943 26944 26945 26946 26947 26948 26949 26950 26951 26952 26953 26954 26955 26956 26957 26958 26959 26960 26961 26962 26963 26964 26965 26966 26967 26968 26969 26970 26971 26972 26973 26974 26975 26976 26977 26978 26979 26980 26981 26982 26983 26984 26985 26986 26987 26988 26989 26990 26991 26992 26993 26994 26995 26996 26997 26998 26999 27000 27001 27002 27003 27004 27005 27006 27007 27008 27009 27010 27011 27012 27013 27014 27015 27016 27017 27018 27019 27020 27021 27022 27023 27024 27025 27026 27027 27028 27029 27030 27031 27032 27033 27034 27035 27036 27037 27038 27039 27040 27041 27042 27043 27044 27045 27046 27047 27048 27049 27050 27051 27052 27053 27054 27055 27056 27057 27058 27059 27060 27061 27062 27063 27064 27065 27066 27067 27068 27069 27070 27071 27072 27073 27074 27075 27076 27077 27078 27079 27080 27081 27082 27083 27084 27085 27086 27087 27088 27089 27090 27091 27092 27093 27094 27095 27096 27097 27098 27099 27100 27101 27102 27103 27104 27105 27106 27107 27108 27109 27110 27111 27112 27113 27114 27115 27116 27117 27118 27119 27120 27121 27122 27123 27124 27125 27126 27127 27128 27129 27130 27131 27132 27133 27134 27135 27136 27137 27138 27139 27140 27141 27142 27143 27144 27145 27146 27147 27148 27149 27150 27151 27152 27153 27154 27155 27156 27157 27158 27159 27160 27161 27162 27163 27164 27165 27166 27167 27168 27169 27170 27171 27172 27173 27174 27175 27176 27177 27178 27179 27180 27181 27182 27183 27184 27185 27186 27187 27188 27189 27190 27191 27192 27193 27194 27195 27196 27197 27198 27199 27200 27201 27202 27203 27204 27205 27206 27207 27208 27209 27210 27211 27212 27213 27214 27215 27216 27217 27218 27219 27220 27221 27222 27223 27224 27225 27226 27227 27228 27229 27230 27231 27232 27233 27234 27235 27236 27237 27238 27239 27240 27241 27242 27243 27244 27245 27246 27247 27248 27249 27250 27251 27252 27253 27254 27255 27256 27257 27258 27259 27260 27261 27262 27263 27264 27265 27266 27267 27268 27269 27270 27271 27272 27273 27274 27275 27276 27277 27278 27279 27280 27281 27282 27283 27284 27285 27286 27287 27288 27289 27290 27291 27292 27293 27294 27295 27296 27297 27298 27299 27300 27301 27302 27303 27304 27305 27306 27307 27308 27309 27310 27311 27312 27313 27314 27315 27316 27317 27318 27319 27320 27321 27322 27323 27324 27325 27326 27327 27328 27329 27330 27331 27332 27333 27334 27335 27336 27337 27338 27339 27340 27341 27342 27343 27344 27345 27346 27347 27348 27349 27350 27351 27352 27353 27354 27355 27356 27357 27358 27359 27360 27361 27362 27363 27364 27365 27366 27367 27368 27369 27370 27371 27372 27373 27374 27375 27376 27377 27378 27379 27380 27381 27382 27383 27384 27385 27386 27387 27388 27389 27390 27391 27392 27393 27394 27395 27396 27397 27398 27399 27400 27401 27402 27403 27404 27405 27406 27407 27408 27409 27410 27411 27412 27413 27414 27415 27416 27417 27418 27419 27420 27421 27422 27423 27424 27425 27426 27427 27428 27429 27430 27431 27432 27433 27434 27435 27436 27437 27438 27439 27440 27441 27442 27443 27444 27445 27446 27447 27448 27449 27450 27451 27452 27453 27454 27455 27456 27457 27458 27459 27460 27461 27462 27463 27464 27465 27466 27467 27468 27469 27470 27471 27472 27473 27474 27475 27476 27477 27478 27479 27480 27481 27482 27483 27484 27485 27486 27487 27488 27489 27490 27491 27492 27493 27494 27495 27496 27497 27498 27499 27500 27501 27502 27503 27504 27505 27506 27507 27508 27509 27510 27511 27512 27513 27514 27515 27516 27517 27518 27519 27520 27521 27522 27523 27524 27525 27526 27527 27528 27529 27530 27531 27532 27533 27534 27535 27536 27537 27538 27539 27540 27541 27542 27543 27544 27545 27546 27547 27548 27549 27550 27551 27552 27553 27554 27555 27556 27557 27558 27559 27560 27561 27562 27563 27564 27565 27566 27567 27568 27569 27570 27571 27572 27573 27574 27575 27576 27577 27578 27579 27580 27581 27582 27583 27584 27585 27586 27587 27588 27589 27590 27591 27592 27593 27594 27595 27596 27597 27598 27599 27600 27601 27602 27603 27604 27605 27606 27607 27608 27609 27610 27611 27612 27613 27614 27615 27616 27617 27618 27619 27620 27621 27622 27623 27624 27625 27626 27627 27628 27629 27630 27631 27632 27633 27634 27635 27636 27637 27638 27639 27640 27641 27642 27643 27644 27645 27646 27647 27648 27649 27650 27651 27652 27653 27654 27655 27656 27657 27658 27659 27660 27661 27662 27663 27664 27665 27666 27667 27668 27669 27670 27671 27672 27673 27674 27675 27676 27677 27678 27679 27680 27681 27682 27683 27684 27685 27686 27687 27688 27689 27690 27691 27692 27693 27694 27695 27696 27697 27698 27699 27700 27701 27702 27703 27704 27705 27706 27707 27708 27709 27710 27711 27712 27713 27714 27715 27716 27717 27718 27719 27720 27721 27722 27723 27724 27725 27726 27727 27728 27729 27730 27731 27732 27733 27734 27735 27736 27737 27738 27739 27740 27741 27742 27743 27744 27745 27746 27747 27748 27749 27750 27751 27752 27753 27754 27755 27756 27757 27758 27759 27760 27761 27762 27763 27764 27765 27766 27767 27768 27769 27770 27771 27772 27773 27774 27775 27776 27777 27778 27779 27780 27781 27782 27783 27784 27785 27786 27787 27788 27789 27790 27791 27792 27793 27794 27795 27796 27797 27798 27799 27800 27801 27802 27803 27804 27805 27806 27807 27808 27809 27810 27811 27812 27813 27814 27815 27816 27817 27818 27819 27820 27821 27822 27823 27824 27825 27826 27827 27828 27829 27830 27831 27832 27833 27834 27835 27836 27837 27838 27839 27840 27841 27842 27843 27844 27845 27846 27847 27848 27849 27850 27851 27852 27853 27854 27855 27856 27857 27858 27859 27860 27861 27862 27863 27864 27865 27866 27867 27868 27869 27870 27871 27872 27873 27874 27875 27876 27877 27878 27879 27880 27881 27882 27883 27884 27885 27886 27887 27888 27889 27890 27891 27892 27893 27894 27895 27896 27897 27898 27899 27900 27901 27902 27903 27904 27905 27906 27907 27908 27909 27910 27911 27912 27913 27914 27915 27916 27917 27918 27919 27920 27921 27922 27923 27924 27925 27926 27927 27928 27929 27930 27931 27932 27933 27934 27935 27936 27937 27938 27939 27940 27941 27942 27943 27944 27945 27946 27947 27948 27949 27950 27951 27952 27953 27954 27955 27956 27957 27958 27959 27960 27961 27962 27963 27964 27965 27966 27967 27968 27969 27970 27971 27972 27973 27974 27975 27976 27977 27978 27979 27980 27981 27982 27983 27984 27985 27986 27987 27988 27989 27990 27991 27992 27993 27994 27995 27996 27997 27998 27999 28000 28001 28002 28003 28004 28005 28006 28007 28008 28009 28010 28011 28012 28013 28014 28015 28016 28017 28018 28019 28020 28021 28022 28023 28024 28025 28026 28027 28028 28029 28030 28031 28032 28033 28034 28035 28036 28037 28038 28039 28040 28041 28042 28043 28044 28045 28046 28047 28048 28049 28050 28051 28052 28053 28054 28055 28056 28057 28058 28059 28060 28061 28062 28063 28064 28065 28066 28067 28068 28069 28070 28071 28072 28073 28074 28075 28076 28077 28078 28079 28080 28081 28082 28083 28084 28085 28086 28087 28088 28089 28090 28091 28092 28093 28094 28095 28096 28097 28098 28099 28100 28101 28102 28103 28104 28105 28106 28107 28108 28109 28110 28111 28112 28113 28114 28115 28116 28117 28118 28119 28120 28121 28122 28123 28124 28125 28126 28127 28128 28129 28130 28131 28132 28133 28134 28135 28136 28137 28138 28139 28140 28141 28142 28143 28144 28145 28146 28147 28148 28149 28150 28151 28152 28153 28154 28155 28156 28157 28158 28159 28160 28161 28162 28163 28164 28165 28166 28167 28168 28169 28170 28171 28172 28173 28174 28175 28176 28177 28178 28179 28180 28181 28182 28183 28184 28185 28186 28187 28188 28189 28190 28191 28192 28193 28194 28195 28196 28197 28198 28199 28200 28201 28202 28203 28204 28205 28206 28207 28208 28209 28210 28211 28212 28213 28214 28215 28216 28217 28218 28219 28220 28221 28222 28223 28224 28225 28226 28227 28228 28229 28230 28231 28232 28233 28234 28235 28236 28237 28238 28239 28240 28241 28242 28243 28244 28245 28246 28247 28248 28249 28250 28251 28252 28253 28254 28255 28256 28257 28258 28259 28260 28261 28262 28263 28264 28265 28266 28267 28268 28269 28270 28271 28272 28273 28274 28275 28276 28277 28278 28279 28280 28281 28282 28283 28284 28285 28286 28287 28288 28289 28290 28291 28292 28293 28294 28295 28296 28297 28298 28299 28300 28301 28302 28303 28304 28305 28306 28307 28308 28309 28310 28311 28312 28313 28314 28315 28316 28317 28318 28319 28320 28321 28322 28323 28324 28325 28326 28327 28328 28329 28330 28331 28332 28333 28334 28335 28336 28337 28338 28339 28340 28341 28342 28343 28344 28345 28346 28347 28348 28349 28350 28351 28352 28353 28354 28355 28356 28357 28358 28359 28360 28361 28362 28363 28364 28365 28366 28367 28368 28369 28370 28371 28372 28373 28374 28375 28376 28377 28378 28379 28380 28381 28382 28383 28384 28385 28386 28387 28388 28389 28390 28391 28392 28393 28394 28395 28396 28397 28398 28399 28400 28401 28402 28403 28404 28405 28406 28407 28408 28409 28410 28411 28412 28413 28414 28415 28416 28417 28418 28419 28420 28421 28422 28423 28424 28425 28426 28427 28428 28429 28430 28431 28432 28433 28434 28435 28436 28437 28438 28439 28440 28441 28442 28443 28444 28445 28446 28447 28448 28449 28450 28451 28452 28453 28454 28455 28456 28457 28458 28459 28460 28461 28462 28463 28464 28465 28466 28467 28468 28469 28470 28471 28472 28473 28474 28475 28476 28477 28478 28479 28480 28481 28482 28483 28484 28485 28486 28487 28488 28489 28490 28491 28492 28493 28494 28495 28496 28497 28498 28499 28500 28501 28502 28503 28504 28505 28506 28507 28508 28509 28510 28511 28512 28513 28514 28515 28516 28517 28518 28519 28520 28521 28522 28523 28524 28525 28526 28527 28528 28529 28530 28531 28532 28533 28534 28535 28536 28537 28538 28539 28540 28541 28542 28543 28544 28545 28546 28547 28548 28549 28550 28551 28552 28553 28554 28555 28556 28557 28558 28559 28560 28561 28562 28563 28564 28565 28566 28567 28568 28569 28570 28571 28572 28573 28574 28575 28576 28577 28578 28579 28580 28581 28582 28583 28584 28585 28586 28587 28588 28589 28590 28591 28592 28593 28594 28595 28596 28597 28598 28599 28600 28601 28602 28603 28604 28605 28606 28607 28608 28609 28610 28611 28612 28613 28614 28615 28616 28617 28618 28619 28620 28621 28622 28623 28624 28625 28626 28627 28628 28629 28630 28631 28632 28633 28634 28635 28636 28637 28638 28639 28640 28641 28642 28643 28644 28645 28646 28647 28648 28649 28650 28651 28652 28653 28654 28655 28656 28657 28658 28659 28660 28661 28662 28663 28664 28665 28666 28667 28668 28669 28670 28671 28672 28673 28674 28675 28676 28677 28678 28679 28680 28681 28682 28683 28684 28685 28686 28687 28688 28689 28690 28691 28692 28693 28694 28695 28696 28697 28698 28699 28700 28701 28702 28703 28704 28705 28706 28707 28708 28709 28710 28711 28712 28713 28714 28715 28716 28717 28718 28719 28720 28721 28722 28723 28724 28725 28726 28727 28728 28729 28730 28731 28732 28733 28734 28735 28736 28737 28738 28739 28740 28741 28742 28743 28744 28745 28746 28747 28748 28749 28750 28751 28752 28753 28754 28755 28756 28757 28758 28759 28760 28761 28762 28763 28764 28765 28766 28767 28768 28769 28770 28771 28772 28773 28774 28775 28776 28777 28778 28779 28780 28781 28782 28783 28784 28785 28786 28787 28788 28789 28790 28791 28792 28793 28794 28795 28796 28797 28798 28799 28800 28801 28802 28803 28804 28805 28806 28807 28808 28809 28810 28811 28812 28813 28814 28815 28816 28817 28818 28819 28820 28821 28822 28823 28824 28825 28826 28827 28828 28829 28830 28831 28832 28833 28834 28835 28836 28837 28838 28839 28840 28841 28842 28843 28844 28845 28846 28847 28848 28849 28850 28851 28852 28853 28854 28855 28856 28857 28858 28859 28860 28861 28862 28863 28864 28865 28866 28867 28868 28869 28870 28871 28872 28873 28874 28875 28876 28877 28878 28879 28880 28881 28882 28883 28884 28885 28886 28887 28888 28889 28890 28891 28892 28893 28894 28895 28896 28897 28898 28899 28900 28901 28902 28903 28904 28905 28906 28907 28908 28909 28910 28911 28912 28913 28914 28915 28916 28917 28918 28919 28920 28921 28922 28923 28924 28925 28926 28927 28928 28929 28930 28931 28932 28933 28934 28935 28936 28937 28938 28939 28940 28941 28942 28943 28944 28945 28946 28947 28948 28949 28950 28951 28952 28953 28954 28955 28956 28957 28958 28959 28960 28961 28962 28963 28964 28965 28966 28967 28968 28969 28970 28971 28972 28973 28974 28975 28976 28977 28978 28979 28980 28981 28982 28983 28984 28985 28986 28987 28988 28989 28990 28991 28992 28993 28994 28995 28996 28997 28998 28999 29000 29001 29002 29003 29004 29005 29006 29007 29008 29009 29010 29011 29012 29013 29014 29015 29016 29017 29018 29019 29020 29021 29022 29023 29024 29025 29026 29027 29028 29029 29030 29031 29032 29033 29034 29035 29036 29037 29038 29039 29040 29041 29042 29043 29044 29045 29046 29047 29048 29049 29050 29051 29052 29053 29054 29055 29056 29057 29058 29059 29060 29061 29062 29063 29064 29065 29066 29067 29068 29069 29070 29071 29072 29073 29074 29075 29076 29077 29078 29079 29080 29081 29082 29083 29084 29085 29086 29087 29088 29089 29090 29091 29092 29093 29094 29095 29096 29097 29098 29099 29100 29101 29102 29103 29104 29105 29106 29107 29108 29109 29110 29111 29112 29113 29114 29115 29116 29117 29118 29119 29120 29121 29122 29123 29124 29125 29126 29127 29128 29129 29130 29131 29132 29133 29134 29135 29136 29137 29138 29139 29140 29141 29142 29143 29144 29145 29146 29147 29148 29149 29150 29151 29152 29153 29154 29155 29156 29157 29158 29159 29160 29161 29162 29163 29164 29165 29166 29167 29168 29169 29170 29171 29172 29173 29174 29175 29176 29177 29178 29179 29180 29181 29182 29183 29184 29185 29186 29187 29188 29189 29190 29191 29192 29193 29194 29195 29196 29197 29198 29199 29200 29201 29202 29203 29204 29205 29206 29207 29208 29209 29210 29211 29212 29213 29214 29215 29216 29217 29218 29219 29220 29221 29222 29223 29224 29225 29226 29227 29228 29229 29230 29231 29232 29233 29234 29235 29236 29237 29238 29239 29240 29241 29242 29243 29244 29245 29246 29247 29248 29249 29250 29251 29252 29253 29254 29255 29256 29257 29258 29259 29260 29261 29262 29263 29264 29265 29266 29267 29268 29269 29270 29271 29272 29273 29274 29275 29276 29277 29278 29279 29280 29281 29282 29283 29284 29285 29286 29287 29288 29289 29290 29291 29292 29293 29294 29295 29296 29297 29298 29299 29300 29301 29302 29303 29304 29305 29306 29307 29308 29309 29310 29311 29312 29313 29314 29315 29316 29317 29318 29319 29320 29321 29322 29323 29324 29325 29326 29327 29328 29329 29330 29331 29332 29333 29334 29335 29336 29337 29338 29339 29340 29341 29342 29343 29344 29345 29346 29347 29348 29349 29350 29351 29352 29353 29354 29355 29356 29357 29358 29359 29360 29361 29362 29363 29364 29365 29366 29367 29368 29369 29370 29371 29372 29373 29374 29375 29376 29377 29378 29379 29380 29381 29382 29383 29384 29385 29386 29387 29388 29389 29390 29391 29392 29393 29394 29395 29396 29397 29398 29399 29400 29401 29402 29403 29404 29405 29406 29407 29408 29409 29410 29411 29412 29413 29414 29415 29416 29417 29418 29419 29420 29421 29422 29423 29424 29425 29426 29427 29428 29429 29430 29431 29432 29433 29434 29435 29436 29437 29438 29439 29440 29441 29442 29443 29444 29445 29446 29447 29448 29449 29450 29451 29452 29453 29454 29455 29456 29457 29458 29459 29460 29461 29462 29463 29464 29465 29466 29467 29468 29469 29470 29471 29472 29473 29474 29475 29476 29477 29478 29479 29480 29481 29482 29483 29484 29485 29486 29487 29488 29489 29490 29491 29492 29493 29494 29495 29496 29497 29498 29499 29500 29501 29502 29503 29504 29505 29506 29507 29508 29509 29510 29511 29512 29513 29514 29515 29516 29517 29518 29519 29520 29521 29522 29523 29524 29525 29526 29527 29528 29529 29530 29531 29532 29533 29534 29535 29536 29537 29538 29539 29540 29541 29542 29543 29544 29545 29546 29547 29548 29549 29550 29551 29552 29553 29554 29555 29556 29557 29558 29559 29560 29561 29562 29563 29564 29565 29566 29567 29568 29569 29570 29571 29572 29573 29574 29575 29576 29577 29578 29579 29580 29581 29582 29583 29584 29585 29586 29587 29588 29589 29590 29591 29592 29593 29594 29595 29596 29597 29598 29599 29600 29601 29602 29603 29604 29605 29606 29607 29608 29609 29610 29611 29612 29613 29614 29615 29616 29617 29618 29619 29620 29621 29622 29623 29624 29625 29626 29627 29628 29629 29630 29631 29632 29633 29634 29635 29636 29637 29638 29639 29640 29641 29642 29643 29644 29645 29646 29647 29648 29649 29650 29651 29652 29653 29654 29655 29656 29657 29658 29659 29660 29661 29662 29663 29664 29665 29666 29667 29668 29669 29670 29671 29672 29673 29674 29675 29676 29677 29678 29679 29680 29681 29682 29683 29684 29685 29686 29687 29688 29689 29690 29691 29692 29693 29694 29695 29696 29697 29698 29699 29700 29701 29702 29703 29704 29705 29706 29707 29708 29709 29710 29711 29712 29713 29714 29715 29716 29717 29718 29719 29720 29721 29722 29723 29724 29725 29726 29727 29728 29729 29730 29731 29732 29733 29734 29735 29736 29737 29738 29739 29740 29741 29742 29743 29744 29745 29746 29747 29748 29749 29750 29751 29752 29753 29754 29755 29756 29757 29758 29759 29760 29761 29762 29763 29764 29765 29766 29767 29768 29769 29770 29771 29772 29773 29774 29775 29776 29777 29778 29779 29780 29781 29782 29783 29784 29785 29786 29787 29788 29789 29790 29791 29792 29793 29794 29795 29796 29797 29798 29799 29800 29801 29802 29803 29804 29805 29806 29807 29808 29809 29810 29811 29812 29813 29814 29815 29816 29817 29818 29819 29820 29821 29822 29823 29824 29825 29826 29827 29828 29829 29830 29831 29832 29833 29834 29835 29836 29837 29838 29839 29840 29841 29842 29843 29844 29845 29846 29847 29848 29849 29850 29851 29852 29853 29854 29855 29856 29857 29858 29859 29860 29861 29862 29863 29864 29865 29866 29867 29868 29869 29870 29871 29872 29873 29874 29875 29876 29877 29878 29879 29880 29881 29882 29883 29884 29885 29886 29887 29888 29889 29890 29891 29892 29893 29894 29895 29896 29897 29898 29899 29900 29901 29902 29903 29904 29905 29906 29907 29908 29909 29910 29911 29912 29913 29914 29915 29916 29917 29918 29919 29920 29921 29922 29923 29924 29925 29926 29927 29928 29929 29930 29931 29932 29933 29934 29935 29936 29937 29938 29939 29940 29941 29942 29943 29944 29945 29946 29947 29948 29949 29950 29951 29952 29953 29954 29955 29956 29957 29958 29959 29960 29961 29962 29963 29964 29965 29966 29967 29968 29969 29970 29971 29972 29973 29974 29975 29976 29977 29978 29979 29980 29981 29982 29983 29984 29985 29986 29987 29988 29989 29990 29991 29992 29993 29994 29995 29996 29997 29998 29999 30000 30001 30002 30003 30004 30005 30006 30007 30008 30009 30010 30011 30012 30013 30014 30015 30016 30017 30018 30019 30020 30021 30022 30023 30024 30025 30026 30027 30028 30029 30030 30031 30032 30033 30034 30035 30036 30037 30038 30039 30040 30041 30042 30043 30044 30045 30046 30047 30048 30049 30050 30051 30052 30053 30054 30055 30056 30057 30058 30059 30060 30061 30062 30063 30064 30065 30066 30067 30068 30069 30070 30071 30072 30073 30074 30075 30076 30077 30078 30079 30080 30081 30082 30083 30084 30085 30086 30087 30088 30089 30090 30091 30092 30093 30094 30095 30096 30097 30098 30099 30100 30101 30102 30103 30104 30105 30106 30107 30108 30109 30110 30111 30112 30113 30114 30115 30116 30117 30118 30119 30120 30121 30122 30123 30124 30125 30126 30127 30128 30129 30130 30131 30132 30133 30134 30135 30136 30137 30138 30139 30140 30141 30142 30143 30144 30145 30146 30147 30148 30149 30150 30151 30152 30153 30154 30155 30156 30157 30158 30159 30160 30161 30162 30163 30164 30165 30166 30167 30168 30169 30170 30171 30172 30173 30174 30175 30176 30177 30178 30179 30180 30181 30182 30183 30184 30185 30186 30187 30188 30189 30190 30191 30192 30193 30194 30195 30196 30197 30198 30199 30200 30201 30202 30203 30204 30205 30206 30207 30208 30209 30210 30211 30212 30213 30214 30215 30216 30217 30218 30219 30220 30221 30222 30223 30224 30225 30226 30227 30228 30229 30230 30231 30232 30233 30234 30235 30236 30237 30238 30239 30240 30241 30242 30243 30244 30245 30246 30247 30248 30249 30250 30251 30252 30253 30254 30255 30256 30257 30258 30259 30260 30261 30262 30263 30264 30265 30266 30267 30268 30269 30270 30271 30272 30273 30274 30275 30276 30277 30278 30279 30280 30281 30282 30283 30284 30285 30286 30287 30288 30289 30290 30291 30292 30293 30294 30295 30296 30297 30298 30299 30300 30301 30302 30303 30304 30305 30306 30307 30308 30309 30310 30311 30312 30313 30314 30315 30316 30317 30318 30319 30320 30321 30322 30323 30324 30325 30326 30327 30328 30329 30330 30331 30332 30333 30334 30335 30336 30337 30338 30339 30340 30341 30342 30343 30344 30345 30346 30347 30348 30349 30350 30351 30352 30353 30354 30355 30356 30357 30358 30359 30360 30361 30362 30363 30364 30365 30366 30367 30368 30369 30370 30371 30372 30373 30374 30375 30376 30377 30378 30379 30380 30381 30382 30383 30384 30385 30386 30387 30388 30389 30390 30391 30392 30393 30394 30395 30396 30397 30398 30399 30400 30401 30402 30403 30404 30405 30406 30407 30408 30409 30410 30411 30412 30413 30414 30415 30416 30417 30418 30419 30420 30421 30422 30423 30424 30425 30426 30427 30428 30429 30430 30431 30432 30433 30434 30435 30436 30437 30438 30439 30440 30441 30442 30443 30444 30445 30446 30447 30448 30449 30450 30451 30452 30453 30454 30455 30456 30457 30458 30459 30460 30461 30462 30463 30464 30465 30466 30467 30468 30469 30470 30471 30472 30473 30474 30475 30476 30477 30478 30479 30480 30481 30482 30483 30484 30485 30486 30487 30488 30489 30490 30491 30492 30493 30494 30495 30496 30497 30498 30499 30500 30501 30502 30503 30504 30505 30506 30507 30508 30509 30510 30511 30512 30513 30514 30515 30516 30517 30518 30519 30520 30521 30522 30523 30524 30525 30526 30527 30528 30529 30530 30531 30532 30533 30534 30535 30536 30537 30538 30539 30540 30541 30542 30543 30544 30545 30546 30547 30548 30549 30550 30551 30552 30553 30554 30555 30556 30557 30558 30559 30560 30561 30562 30563 30564 30565 30566 30567 30568 30569 30570 30571 30572 30573 30574 30575 30576 30577 30578 30579 30580 30581 30582 30583 30584 30585 30586 30587 30588 30589 30590 30591 30592 30593 30594 30595 30596 30597 30598 30599 30600 30601 30602 30603 30604 30605 30606 30607 30608 30609 30610 30611 30612 30613 30614 30615 30616 30617 30618 30619 30620 30621 30622 30623 30624 30625 30626 30627 30628 30629 30630 30631 30632 30633 30634 30635 30636 30637 30638 30639 30640 30641 30642 30643 30644 30645 30646 30647 30648 30649 30650 30651 30652 30653 30654 30655 30656 30657 30658 30659 30660 30661 30662 30663 30664 30665 30666 30667 30668 30669 30670 30671 30672 30673 30674 30675 30676 30677 30678 30679 30680 30681 30682 30683 30684 30685 30686 30687 30688 30689 30690 30691 30692 30693 30694 30695 30696 30697 30698 30699 30700 30701 30702 30703 30704 30705 30706 30707 30708 30709 30710 30711 30712 30713 30714 30715 30716 30717 30718 30719 30720 30721 30722 30723 30724 30725 30726 30727 30728 30729 30730 30731 30732 30733 30734 30735 30736 30737 30738 30739 30740 30741 30742 30743 30744 30745 30746 30747 30748 30749 30750 30751 30752 30753 30754 30755 30756 30757 30758 30759 30760 30761 30762 30763 30764 30765 30766 30767 30768 30769 30770 30771 30772 30773 30774 30775 30776 30777 30778 30779 30780 30781 30782 30783 30784 30785 30786 30787 30788 30789 30790 30791 30792 30793 30794 30795 30796 30797 30798 30799 30800 30801 30802 30803 30804 30805 30806 30807 30808 30809 30810 30811 30812 30813 30814 30815 30816 30817 30818 30819 30820 30821 30822 30823 30824 30825 30826 30827 30828 30829 30830 30831 30832 30833 30834 30835 30836 30837 30838 30839 30840 30841 30842 30843 30844 30845 30846 30847 30848 30849 30850 30851 30852 30853 30854 30855 30856 30857 30858 30859 30860 30861 30862 30863 30864 30865 30866 30867 30868 30869 30870 30871 30872 30873 30874 30875 30876 30877 30878 30879 30880 30881 30882 30883 30884 30885 30886 30887 30888 30889 30890 30891 30892 30893 30894 30895 30896 30897 30898 30899 30900 30901 30902 30903 30904 30905 30906 30907 30908 30909 30910 30911 30912 30913 30914 30915 30916 30917 30918 30919 30920 30921 30922 30923 30924 30925 30926 30927 30928 30929 30930 30931 30932 30933 30934 30935 30936 30937 30938 30939 30940 30941 30942 30943 30944 30945 30946 30947 30948 30949 30950 30951 30952 30953 30954 30955 30956 30957 30958 30959 30960 30961 30962 30963 30964 30965 30966 30967 30968 30969 30970 30971 30972 30973 30974 30975 30976 30977 30978 30979 30980 30981 30982 30983 30984 30985 30986 30987 30988 30989 30990 30991 30992 30993 30994 30995 30996 30997 30998 30999 31000 31001 31002 31003 31004 31005 31006 31007 31008 31009 31010 31011 31012 31013 31014 31015 31016 31017 31018 31019 31020 31021 31022 31023 31024 31025 31026 31027 31028 31029 31030 31031 31032 31033 31034 31035 31036 31037 31038 31039 31040 31041 31042 31043 31044 31045 31046 31047 31048 31049 31050 31051 31052 31053 31054 31055 31056 31057 31058 31059 31060 31061 31062 31063 31064 31065 31066 31067 31068 31069 31070 31071 31072 31073 31074 31075 31076 31077 31078 31079 31080 31081 31082 31083 31084 31085 31086 31087 31088 31089 31090 31091 31092 31093 31094 31095 31096 31097 31098 31099 31100 31101 31102 31103 31104 31105 31106 31107 31108 31109 31110 31111 31112 31113 31114 31115 31116 31117 31118 31119 31120 31121 31122 31123 31124 31125 31126 31127 31128 31129 31130 31131 31132 31133 31134 31135 31136 31137 31138 31139 31140 31141 31142 31143 31144 31145 31146 31147 31148 31149 31150 31151 31152 31153 31154 31155 31156 31157 31158 31159 31160 31161 31162 31163 31164 31165 31166 31167 31168 31169 31170 31171 31172 31173 31174 31175 31176 31177 31178 31179 31180 31181 31182 31183 31184 31185 31186 31187 31188 31189 31190 31191 31192 31193 31194 31195 31196 31197 31198 31199 31200 31201 31202 31203 31204 31205 31206 31207 31208 31209 31210 31211 31212 31213 31214 31215 31216 31217 31218 31219 31220 31221 31222 31223 31224 31225 31226 31227 31228 31229 31230 31231 31232 31233 31234 31235 31236 31237 31238 31239 31240 31241 31242 31243 31244 31245 31246 31247 31248 31249 31250 31251 31252 31253 31254 31255 31256 31257 31258 31259 31260 31261 31262 31263 31264 31265 31266 31267 31268 31269 31270 31271 31272 31273 31274 31275 31276 31277 31278 31279 31280 31281 31282 31283 31284 31285 31286 31287 31288 31289 31290 31291 31292 31293 31294 31295 31296 31297 31298 31299 31300 31301 31302 31303 31304 31305 31306 31307 31308 31309 31310 31311 31312 31313 31314 31315 31316 31317 31318 31319 31320 31321 31322 31323 31324 31325 31326 31327 31328 31329 31330 31331 31332 31333 31334 31335 31336 31337 31338 31339 31340 31341 31342 31343 31344 31345 31346 31347 31348 31349 31350 31351 31352 31353 31354 31355 31356 31357 31358 31359 31360 31361 31362 31363 31364 31365 31366 31367 31368 31369 31370 31371 31372 31373 31374 31375 31376 31377 31378 31379 31380 31381 31382 31383 31384 31385 31386 31387 31388 31389 31390 31391 31392 31393 31394 31395 31396 31397 31398 31399 31400 31401 31402 31403 31404 31405 31406 31407 31408 31409 31410 31411 31412 31413 31414 31415 31416 31417 31418 31419 31420 31421 31422 31423 31424 31425 31426 31427 31428 31429 31430 31431 31432 31433 31434 31435 31436 31437 31438 31439 31440 31441 31442 31443 31444 31445 31446 31447 31448 31449 31450 31451 31452 31453 31454 31455 31456 31457 31458 31459 31460 31461 31462 31463 31464 31465 31466 31467 31468 31469 31470 31471 31472 31473 31474 31475 31476 31477 31478 31479 31480 31481 31482 31483 31484 31485 31486 31487 31488 31489 31490 31491 31492 31493 31494 31495 31496 31497 31498 31499 31500 31501 31502 31503 31504 31505 31506 31507 31508 31509 31510 31511 31512 31513 31514 31515 31516 31517 31518 31519 31520 31521 31522 31523 31524 31525 31526 31527 31528 31529 31530 31531 31532 31533 31534 31535 31536 31537 31538 31539 31540 31541 31542 31543 31544 31545 31546 31547 31548 31549 31550 31551 31552 31553 31554 31555 31556 31557 31558 31559 31560 31561 31562 31563 31564 31565 31566 31567 31568 31569 31570 31571 31572 31573 31574 31575 31576 31577 31578 31579 31580 31581 31582 31583 31584 31585 31586 31587 31588 31589 31590 31591 31592 31593 31594 31595 31596 31597 31598 31599 31600 31601 31602 31603 31604 31605 31606 31607 31608 31609 31610 31611 31612 31613 31614 31615 31616 31617 31618 31619 31620 31621 31622 31623 31624 31625 31626 31627 31628 31629 31630 31631 31632 31633 31634 31635 31636 31637 31638 31639 31640 31641 31642 31643 31644 31645 31646 31647 31648 31649 31650 31651 31652 31653 31654 31655 31656 31657 31658 31659 31660 31661 31662 31663 31664 31665 31666 31667 31668 31669 31670 31671 31672 31673 31674 31675 31676 31677 31678 31679 31680 31681 31682 31683 31684 31685 31686 31687 31688 31689 31690 31691 31692 31693 31694 31695 31696 31697 31698 31699 31700 31701 31702 31703 31704 31705 31706 31707 31708 31709 31710 31711 31712 31713 31714 31715 31716 31717 31718 31719 31720 31721 31722 31723 31724 31725 31726 31727 31728 31729 31730 31731 31732 31733 31734 31735 31736 31737 31738 31739 31740 31741 31742 31743 31744 31745 31746 31747 31748 31749 31750 31751 31752 31753 31754 31755 31756 31757 31758 31759 31760 31761 31762 31763 31764 31765 31766 31767 31768 31769 31770 31771 31772 31773 31774 31775 31776 31777 31778 31779 31780 31781 31782 31783 31784 31785 31786 31787 31788 31789 31790 31791 31792 31793 31794 31795 31796 31797 31798 31799 31800 31801 31802 31803 31804 31805 31806 31807 31808 31809 31810 31811 31812 31813 31814 31815 31816 31817 31818 31819 31820 31821 31822 31823 31824 31825 31826 31827 31828 31829 31830 31831 31832 31833 31834 31835 31836 31837 31838 31839 31840 31841 31842 31843 31844 31845 31846 31847 31848 31849 31850 31851 31852 31853 31854 31855 31856 31857 31858 31859 31860 31861 31862 31863 31864 31865 31866 31867 31868 31869 31870 31871 31872 31873 31874 31875 31876 31877 31878 31879 31880 31881 31882 31883 31884 31885 31886 31887 31888 31889 31890 31891 31892 31893 31894 31895 31896 31897 31898 31899 31900 31901 31902 31903 31904 31905 31906 31907 31908 31909 31910 31911 31912 31913 31914 31915 31916 31917 31918 31919 31920 31921 31922 31923 31924 31925 31926 31927 31928 31929 31930 31931 31932 31933 31934 31935 31936 31937 31938 31939 31940 31941 31942 31943 31944 31945 31946 31947 31948 31949 31950 31951 31952 31953 31954 31955 31956 31957 31958 31959 31960 31961 31962 31963 31964 31965 31966 31967 31968 31969 31970 31971 31972 31973 31974 31975 31976 31977 31978 31979 31980 31981 31982 31983 31984 31985 31986 31987 31988 31989 31990 31991 31992 31993 31994 31995 31996 31997 31998 31999 32000 32001 32002 32003 32004 32005 32006 32007 32008 32009 32010 32011 32012 32013 32014 32015 32016 32017 32018 32019 32020 32021 32022 32023 32024 32025 32026 32027 32028 32029 32030 32031 32032 32033 32034 32035 32036 32037 32038 32039 32040 32041 32042 32043 32044 32045 32046 32047 32048 32049 32050 32051 32052 32053 32054 32055 32056 32057 32058 32059 32060 32061 32062 32063 32064 32065 32066 32067 32068 32069 32070 32071 32072 32073 32074 32075 32076 32077 32078 32079 32080 32081 32082 32083 32084 32085 32086 32087 32088 32089 32090 32091 32092 32093 32094 32095 32096 32097 32098 32099 32100 32101 32102 32103 32104 32105 32106 32107 32108 32109 32110 32111 32112 32113 32114 32115 32116 32117 32118 32119 32120 32121 32122 32123 32124 32125 32126 32127 32128 32129 32130 32131 32132 32133 32134 32135 32136 32137 32138 32139 32140 32141 32142 32143 32144 32145 32146 32147 32148 32149 32150 32151 32152 32153 32154 32155 32156 32157 32158 32159 32160 32161 32162 32163 32164 32165 32166 32167 32168 32169 32170 32171 32172 32173 32174 32175 32176 32177 32178 32179 32180 32181 32182 32183 32184 32185 32186 32187 32188 32189 32190 32191 32192 32193 32194 32195 32196 32197 32198 32199 32200 32201 32202 32203 32204 32205 32206 32207 32208 32209 32210 32211 32212 32213 32214 32215 32216 32217 32218 32219 32220 32221 32222 32223 32224 32225 32226 32227 32228 32229 32230 32231 32232 32233 32234 32235 32236 32237 32238 32239 32240 32241 32242 32243 32244 32245 32246 32247 32248 32249 32250 32251 32252 32253 32254 32255 32256 32257 32258 32259 32260 32261 32262 32263 32264 32265 32266 32267 32268 32269 32270 32271 32272 32273 32274 32275 32276 32277 32278 32279 32280 32281 32282 32283 32284 32285 32286 32287 32288 32289 32290 32291 32292 32293 32294 32295 32296 32297 32298 32299 32300 32301 32302 32303 32304 32305 32306 32307 32308 32309 32310 32311 32312 32313 32314 32315 32316 32317 32318 32319 32320 32321 32322 32323 32324 32325 32326 32327 32328 32329 32330 32331 32332 32333 32334 32335 32336 32337 32338 32339 32340 32341 32342 32343 32344 32345 32346 32347 32348 32349 32350 32351 32352 32353 32354 32355 32356 32357 32358 32359 32360 32361 32362 32363 32364 32365 32366 32367 32368 32369 32370 32371 32372 32373 32374 32375 32376 32377 32378 32379 32380 32381 32382 32383 32384 32385 32386 32387 32388 32389 32390 32391 32392 32393 32394 32395 32396 32397 32398 32399 32400 32401 32402 32403 32404 32405 32406 32407 32408 32409 32410 32411 32412 32413 32414 32415 32416 32417 32418 32419 32420 32421 32422 32423 32424 32425 32426 32427 32428 32429 32430 32431 32432 32433 32434 32435 32436 32437 32438 32439 32440 32441 32442 32443 32444 32445 32446 32447 32448 32449 32450 32451 32452 32453 32454 32455 32456 32457 32458 32459 32460 32461 32462 32463 32464 32465 32466 32467 32468 32469 32470 32471 32472 32473 32474 32475 32476 32477 32478 32479 32480 32481 32482 32483 32484 32485 32486 32487 32488 32489 32490 32491 32492 32493 32494 32495 32496 32497 32498 32499 32500 32501 32502 32503 32504 32505 32506 32507 32508 32509 32510 32511 32512 32513 32514 32515 32516 32517 32518 32519 32520 32521 32522 32523 32524 32525 32526 32527 32528 32529 32530 32531 32532 32533 32534 32535 32536 32537 32538 32539 32540 32541 32542 32543 32544 32545 32546 32547 32548 32549 32550 32551 32552 32553 32554 32555 32556 32557 32558 32559 32560 32561 32562 32563 32564 32565 32566 32567 32568 32569 32570 32571 32572 32573 32574 32575 32576 32577 32578 32579 32580 32581 32582 32583 32584 32585 32586 32587 32588 32589 32590 32591 32592 32593 32594 32595 32596 32597 32598 32599 32600 32601 32602 32603 32604 32605 32606 32607 32608 32609 32610 32611 32612 32613 32614 32615 32616 32617 32618 32619 32620 32621 32622 32623 32624 32625 32626 32627 32628 32629 32630 32631 32632 32633 32634 32635 32636 32637 32638 32639 32640 32641 32642 32643 32644 32645 32646 32647 32648 32649 32650 32651 32652 32653 32654 32655 32656 32657 32658 32659 32660 32661 32662 32663 32664 32665 32666 32667 32668 32669 32670 32671 32672 32673 32674 32675 32676 32677 32678 32679 32680 32681 32682 32683 32684 32685 32686 32687 32688 32689 32690 32691 32692 32693 32694 32695 32696 32697 32698 32699 32700 32701 32702 32703 32704 32705 32706 32707 32708 32709 32710 32711 32712 32713 32714 32715 32716 32717 32718 32719 32720 32721 32722 32723 32724 32725 32726 32727 32728 32729 32730 32731 32732 32733 32734 32735 32736 32737 32738 32739 32740 32741 32742 32743 32744 32745 32746 32747 32748 32749 32750 32751 32752 32753 32754 32755 32756 32757 32758 32759 32760 32761 32762 32763 32764 32765 32766 32767 32768 32769 32770 32771 32772 32773 32774 32775 32776 32777 32778 32779 32780 32781 32782 32783 32784 32785 32786 32787 32788 32789 32790 32791 32792 32793 32794 32795 32796 32797 32798 32799 32800 32801 32802 32803 32804 32805 32806 32807 32808 32809 32810 32811 32812 32813 32814 32815 32816 32817 32818 32819 32820 32821 32822 32823 32824 32825 32826 32827 32828 32829 32830 32831 32832 32833 32834 32835 32836 32837 32838 32839 32840 32841 32842 32843 32844 32845 32846 32847 32848 32849 32850 32851 32852 32853 32854 32855 32856 32857 32858 32859 32860 32861 32862 32863 32864 32865 32866 32867 32868 32869 32870 32871 32872 32873 32874 32875 32876 32877 32878 32879 32880 32881 32882 32883 32884 32885 32886 32887 32888 32889 32890 32891 32892 32893 32894 32895 32896 32897 32898 32899 32900 32901 32902 32903 32904 32905 32906 32907 32908 32909 32910 32911 32912 32913 32914 32915 32916 32917 32918 32919 32920 32921 32922 32923 32924 32925 32926 32927 32928 32929 32930 32931 32932 32933 32934 32935 32936 32937 32938 32939 32940 32941 32942 32943 32944 32945 32946 32947 32948 32949 32950 32951 32952 32953 32954 32955 32956 32957 32958 32959 32960 32961 32962 32963 32964 32965 32966 32967 32968 32969 32970 32971 32972 32973 32974 32975 32976 32977 32978 32979 32980 32981 32982 32983 32984 32985 32986 32987 32988 32989 32990 32991 32992 32993 32994 32995 32996 32997 32998 32999 33000 33001 33002 33003 33004 33005 33006 33007 33008 33009 33010 33011 33012 33013 33014 33015 33016 33017 33018 33019 33020 33021 33022 33023 33024 33025 33026 33027 33028 33029 33030 33031 33032 33033 33034 33035 33036 33037 33038 33039 33040 33041 33042 33043 33044 33045 33046 33047 33048 33049 33050 33051 33052 33053 33054 33055 33056 33057 33058 33059 33060 33061 33062 33063 33064 33065 33066 33067 33068 33069 33070 33071 33072 33073 33074 33075 33076 33077 33078 33079 33080 33081 33082 33083 33084 33085 33086 33087 33088 33089 33090 33091 33092 33093 33094 33095 33096 33097 33098 33099 33100 33101 33102 33103 33104 33105 33106 33107 33108 33109 33110 33111 33112 33113 33114 33115 33116 33117 33118 33119 33120 33121 33122 33123 33124 33125 33126 33127 33128 33129 33130 33131 33132 33133 33134 33135 33136 33137 33138 33139 33140 33141 33142 33143 33144 33145 33146 33147 33148 33149 33150 33151 33152 33153 33154 33155 33156 33157 33158 33159 33160 33161 33162 33163 33164 33165 33166 33167 33168 33169 33170 33171 33172 33173 33174 33175 33176 33177 33178 33179 33180 33181 33182 33183 33184 33185 33186 33187 33188 33189 33190 33191 33192 33193 33194 33195 33196 33197 33198 33199 33200 33201 33202 33203 33204 33205 33206 33207 33208 33209 33210 33211 33212 33213 33214 33215 33216 33217 33218 33219 33220 33221 33222 33223 33224 33225 33226 33227 33228 33229 33230 33231 33232 33233 33234 33235 33236 33237 33238 33239 33240 33241 33242 33243 33244 33245 33246 33247 33248 33249 33250 33251 33252 33253 33254 33255 33256 33257 33258 33259 33260 33261 33262 33263 33264 33265 33266 33267 33268 33269 33270 33271 33272 33273 33274 33275 33276 33277 33278 33279 33280 33281 33282 33283 33284 33285 33286 33287 33288 33289 33290 33291 33292 33293 33294 33295 33296 33297 33298 33299 33300 33301 33302 33303 33304 33305 33306 33307 33308 33309 33310 33311 33312 33313 33314 33315 33316 33317 33318 33319 33320 33321 33322 33323 33324 33325 33326 33327 33328 33329 33330 33331 33332 33333 33334 33335 33336 33337 33338 33339 33340 33341 33342 33343 33344 33345 33346 33347 33348 33349 33350 33351 33352 33353 33354 33355 33356 33357 33358 33359 33360 33361 33362 33363 33364 33365 33366 33367 33368 33369 33370 33371 33372 33373 33374 33375 33376 33377 33378 33379 33380 33381 33382 33383 33384 33385 33386 33387 33388 33389 33390 33391 33392 33393 33394 33395 33396 33397 33398 33399 33400 33401 33402 33403 33404 33405 33406 33407 33408 33409 33410 33411 33412 33413 33414 33415 33416 33417 33418 33419 33420 33421 33422 33423 33424 33425 33426 33427 33428 33429 33430 33431 33432 33433 33434 33435 33436 33437 33438 33439 33440 33441 33442 33443 33444 33445 33446 33447 33448 33449 33450 33451 33452 33453 33454 33455 33456 33457 33458 33459 33460 33461 33462 33463 33464 33465 33466 33467 33468 33469 33470 33471 33472 33473 33474 33475 33476 33477 33478 33479 33480 33481 33482 33483 33484 33485 33486 33487 33488 33489 33490 33491 33492 33493 33494 33495 33496 33497 33498 33499 33500 33501 33502 33503 33504 33505 33506 33507 33508 33509 33510 33511 33512 33513 33514 33515 33516 33517 33518 33519 33520 33521 33522 33523 33524 33525 33526 33527 33528 33529 33530 33531 33532 33533 33534 33535 33536 33537 33538 33539 33540 33541 33542 33543 33544 33545 33546 33547 33548 33549 33550 33551 33552 33553 33554 33555 33556 33557 33558 33559 33560 33561 33562 33563 33564 33565 33566 33567 33568 33569 33570 33571 33572 33573 33574 33575 33576 33577 33578 33579 33580 33581 33582 33583 33584 33585 33586 33587 33588 33589 33590 33591 33592 33593 33594 33595 33596 33597 33598 33599 33600 33601 33602 33603 33604 33605 33606 33607 33608 33609 33610 33611 33612 33613 33614 33615 33616 33617 33618 33619 33620 33621 33622 33623 33624 33625 33626 33627 33628 33629 33630 33631 33632 33633 33634 33635 33636 33637 33638 33639 33640 33641 33642 33643 33644 33645 33646 33647 33648 33649 33650 33651 33652 33653 33654 33655 33656 33657 33658 33659 33660 33661 33662 33663 33664 33665 33666 33667 33668 33669 33670 33671 33672 33673 33674 33675 33676 33677 33678 33679 33680 33681 33682 33683 33684 33685 33686 33687 33688 33689 33690 33691 33692 33693 33694 33695 33696 33697 33698 33699 33700 33701 33702 33703 33704 33705 33706 33707 33708 33709 33710 33711 33712 33713 33714 33715 33716 33717 33718 33719 33720 33721 33722 33723 33724 33725 33726 33727 33728 33729 33730 33731 33732 33733 33734 33735 33736 33737 33738 33739 33740 33741 33742 33743 33744 33745 33746 33747 33748 33749 33750 33751 33752 33753 33754 33755 33756 33757 33758 33759 33760 33761 33762 33763 33764 33765 33766 33767 33768 33769 33770 33771 33772 33773 33774 33775 33776 33777 33778 33779 33780 33781 33782 33783 33784 33785 33786 33787 33788 33789 33790 33791 33792 33793 33794 33795 33796 33797 33798 33799 33800 33801 33802 33803 33804 33805 33806 33807 33808 33809 33810 33811 33812 33813 33814 33815 33816 33817 33818 33819 33820 33821 33822 33823 33824 33825 33826 33827 33828 33829 33830 33831 33832 33833 33834 33835 33836 33837 33838 33839 33840 33841 33842 33843 33844 33845 33846 33847 33848 33849 33850 33851 33852 33853 33854 33855 33856 33857 33858 33859 33860 33861 33862 33863 33864 33865 33866 33867 33868 33869 33870 33871 33872 33873 33874 33875 33876 33877 33878 33879 33880 33881 33882 33883 33884 33885 33886 33887 33888 33889 33890 33891 33892 33893 33894 33895 33896 33897 33898 33899 33900 33901 33902 33903 33904 33905 33906 33907 33908 33909 33910 33911 33912 33913 33914 33915 33916 33917 33918 33919 33920 33921 33922 33923 33924 33925 33926 33927 33928 33929 33930 33931 33932 33933 33934 33935 33936 33937 33938 33939 33940 33941 33942 33943 33944 33945 33946 33947 33948 33949 33950 33951 33952 33953 33954 33955 33956 33957 33958 33959 33960 33961 33962 33963 33964 33965 33966 33967 33968 33969 33970 33971 33972 33973 33974 33975 33976 33977 33978 33979 33980 33981 33982 33983 33984 33985 33986 33987 33988 33989 33990 33991 33992 33993 33994 33995 33996 33997 33998 33999 34000 34001 34002 34003 34004 34005 34006 34007 34008 34009 34010 34011 34012 34013 34014 34015 34016 34017 34018 34019 34020 34021 34022 34023 34024 34025 34026 34027 34028 34029 34030 34031 34032 34033 34034 34035 34036 34037 34038 34039 34040 34041 34042 34043 34044 34045 34046 34047 34048 34049 34050 34051 34052 34053 34054 34055 34056 34057 34058 34059 34060 34061 34062 34063 34064 34065 34066 34067 34068 34069 34070 34071 34072 34073 34074 34075 34076 34077 34078 34079 34080 34081 34082 34083 34084 34085 34086 34087 34088 34089 34090 34091 34092 34093 34094 34095 34096 34097 34098 34099 34100 34101 34102 34103 34104 34105 34106 34107 34108 34109 34110 34111 34112 34113 34114 34115 34116 34117 34118 34119 34120 34121 34122 34123 34124 34125 34126 34127 34128 34129 34130 34131 34132 34133 34134 34135 34136 34137 34138 34139 34140 34141 34142 34143 34144 34145 34146 34147 34148 34149 34150 34151 34152 34153 34154 34155 34156 34157 34158 34159 34160 34161 34162 34163 34164 34165 34166 34167 34168 34169 34170 34171 34172 34173 34174 34175 34176 34177 34178 34179 34180 34181 34182 34183 34184 34185 34186 34187 34188 34189 34190 34191 34192 34193 34194 34195 34196 34197 34198 34199 34200 34201 34202 34203 34204 34205 34206 34207 34208 34209 34210 34211 34212 34213 34214 34215 34216 34217 34218 34219 34220 34221 34222 34223 34224 34225 34226 34227 34228 34229 34230 34231 34232 34233 34234 34235 34236 34237 34238 34239 34240 34241 34242 34243 34244 34245 34246 34247 34248 34249 34250 34251 34252 34253 34254 34255 34256 34257 34258 34259 34260 34261 34262 34263 34264 34265 34266 34267 34268 34269 34270 34271 34272 34273 34274 34275 34276 34277 34278 34279 34280 34281 34282 34283 34284 34285 34286 34287 34288 34289 34290 34291 34292 34293 34294 34295 34296 34297 34298 34299 34300 34301 34302 34303 34304 34305 34306 34307 34308 34309 34310 34311 34312 34313 34314 34315 34316 34317 34318 34319 34320 34321 34322 34323 34324 34325 34326 34327 34328 34329 34330 34331 34332 34333 34334 34335 34336 34337 34338 34339 34340 34341 34342 34343 34344 34345 34346 34347 34348 34349 34350 34351 34352 34353 34354 34355 34356 34357 34358 34359 34360 34361 34362 34363 34364 34365 34366 34367 34368 34369 34370 34371 34372 34373 34374 34375 34376 34377 34378 34379 34380 34381 34382 34383 34384 34385 34386 34387 34388 34389 34390 34391 34392 34393 34394 34395 34396 34397 34398 34399 34400 34401 34402 34403 34404 34405 34406 34407 34408 34409 34410 34411 34412 34413 34414 34415 34416 34417 34418 34419 34420 34421 34422 34423 34424 34425 34426 34427 34428 34429 34430 34431 34432 34433 34434 34435 34436 34437 34438 34439 34440 34441 34442 34443 34444 34445 34446 34447 34448 34449 34450 34451 34452 34453 34454 34455 34456 34457 34458 34459 34460 34461 34462 34463 34464 34465 34466 34467 34468 34469 34470 34471 34472 34473 34474 34475 34476 34477 34478 34479 34480 34481 34482 34483 34484 34485 34486 34487 34488 34489 34490 34491 34492 34493 34494 34495 34496 34497 34498 34499 34500 34501 34502 34503 34504 34505 34506 34507 34508 34509 34510 34511 34512 34513 34514 34515 34516 34517 34518 34519 34520 34521 34522 34523 34524 34525 34526 34527 34528 34529 34530 34531 34532 34533 34534 34535 34536 34537 34538 34539 34540 34541 34542 34543 34544 34545 34546 34547 34548 34549 34550 34551 34552 34553 34554 34555 34556 34557 34558 34559 34560 34561 34562 34563 34564 34565 34566 34567 34568 34569 34570 34571 34572 34573 34574 34575 34576 34577 34578 34579 34580 34581 34582 34583 34584 34585 34586 34587 34588 34589 34590 34591 34592 34593 34594 34595 34596 34597 34598 34599 34600 34601 34602 34603 34604 34605 34606 34607 34608 34609 34610 34611 34612 34613 34614 34615 34616 34617 34618 34619 34620 34621 34622 34623 34624 34625 34626 34627 34628 34629 34630 34631 34632 34633 34634 34635 34636 34637 34638 34639 34640 34641 34642 34643 34644 34645 34646 34647 34648 34649 34650 34651 34652 34653 34654 34655 34656 34657 34658 34659 34660 34661 34662 34663 34664 34665 34666 34667 34668 34669 34670 34671 34672 34673 34674 34675 34676 34677 34678 34679 34680 34681 34682 34683 34684 34685 34686 34687 34688 34689 34690 34691 34692 34693 34694 34695 34696 34697 34698 34699 34700 34701 34702 34703 34704 34705 34706 34707 34708 34709 34710 34711 34712 34713 34714 34715 34716 34717 34718 34719 34720 34721 34722 34723 34724 34725 34726 34727 34728 34729 34730 34731 34732 34733 34734 34735 34736 34737 34738 34739 34740 34741 34742 34743 34744 34745 34746 34747 34748 34749 34750 34751 34752 34753 34754 34755 34756 34757 34758 34759 34760 34761 34762 34763 34764 34765 34766 34767 34768 34769 34770 34771 34772 34773 34774 34775 34776 34777 34778 34779 34780 34781 34782 34783 34784 34785 34786 34787 34788 34789 34790 34791 34792 34793 34794 34795 34796 34797 34798 34799 34800 34801 34802 34803 34804 34805 34806 34807 34808 34809 34810 34811 34812 34813 34814 34815 34816 34817 34818 34819 34820 34821 34822 34823 34824 34825 34826 34827 34828 34829 34830 34831 34832 34833 34834 34835 34836 34837 34838 34839 34840 34841 34842 34843 34844 34845 34846 34847 34848 34849 34850 34851 34852 34853 34854 34855 34856 34857 34858 34859 34860 34861 34862 34863 34864 34865 34866 34867 34868 34869 34870 34871 34872 34873 34874 34875 34876 34877 34878 34879 34880 34881 34882 34883 34884 34885 34886 34887 34888 34889 34890 34891 34892 34893 34894 34895 34896 34897 34898 34899 34900 34901 34902 34903 34904 34905 34906 34907 34908 34909 34910 34911 34912 34913 34914 34915 34916 34917 34918 34919 34920 34921 34922 34923 34924 34925 34926 34927 34928 34929 34930 34931 34932 34933 34934 34935 34936 34937 34938 34939 34940 34941 34942 34943 34944 34945 34946 34947 34948 34949 34950 34951 34952 34953 34954 34955 34956 34957 34958 34959 34960 34961 34962 34963 34964 34965 34966 34967 34968 34969 34970 34971 34972 34973 34974 34975 34976 34977 34978 34979 34980 34981 34982 34983 34984 34985 34986 34987 34988 34989 34990 34991 34992 34993 34994 34995 34996 34997 34998 34999 35000 35001 35002 35003 35004 35005 35006 35007 35008 35009 35010 35011 35012 35013 35014 35015 35016 35017 35018 35019 35020 35021 35022 35023 35024 35025 35026 35027 35028 35029 35030 35031 35032 35033 35034 35035 35036 35037 35038 35039 35040 35041 35042 35043 35044 35045 35046 35047 35048 35049 35050 35051 35052 35053 35054 35055 35056 35057 35058 35059 35060 35061 35062 35063 35064 35065 35066 35067 35068 35069 35070 35071 35072 35073 35074 35075 35076 35077 35078 35079 35080 35081 35082 35083 35084 35085 35086 35087 35088 35089 35090 35091 35092 35093 35094 35095 35096 35097 35098 35099 35100 35101 35102 35103 35104 35105 35106 35107 35108 35109 35110 35111 35112 35113 35114 35115 35116 35117 35118 35119 35120 35121 35122 35123 35124 35125 35126 35127 35128 35129 35130 35131 35132 35133 35134 35135 35136 35137 35138 35139 35140 35141 35142 35143 35144 35145 35146 35147 35148 35149 35150 35151 35152 35153 35154 35155 35156 35157 35158 35159 35160 35161 35162 35163 35164 35165 35166 35167 35168 35169 35170 35171 35172 35173 35174 35175 35176 35177 35178 35179 35180 35181 35182 35183 35184 35185 35186 35187 35188 35189 35190 35191 35192 35193 35194 35195 35196 35197 35198 35199 35200 35201 35202 35203 35204 35205 35206 35207 35208 35209 35210 35211 35212 35213 35214 35215 35216 35217 35218 35219 35220 35221 35222 35223 35224 35225 35226 35227 35228 35229 35230 35231 35232 35233 35234 35235 35236 35237 35238 35239 35240 35241 35242 35243 35244 35245 35246 35247 35248 35249 35250 35251 35252 35253 35254 35255 35256 35257 35258 35259 35260 35261 35262 35263 35264 35265 35266 35267 35268 35269 35270 35271 35272 35273 35274 35275 35276 35277 35278 35279 35280 35281 35282 35283 35284 35285 35286 35287 35288 35289 35290 35291 35292 35293 35294 35295 35296 35297 35298 35299 35300 35301 35302 35303 35304 35305 35306 35307 35308 35309 35310 35311 35312 35313 35314 35315 35316 35317 35318 35319 35320 35321 35322 35323 35324 35325 35326 35327 35328 35329 35330 35331 35332 35333 35334 35335 35336 35337 35338 35339 35340 35341 35342 35343 35344 35345 35346 35347 35348 35349 35350 35351 35352 35353 35354 35355 35356 35357 35358 35359 35360 35361 35362 35363 35364 35365 35366 35367 35368 35369 35370 35371 35372 35373 35374 35375 35376 35377 35378 35379 35380 35381 35382 35383 35384 35385 35386 35387 35388 35389 35390 35391 35392 35393 35394 35395 35396 35397 35398 35399 35400 35401 35402 35403 35404 35405 35406 35407 35408 35409 35410 35411 35412 35413 35414 35415 35416 35417 35418 35419 35420 35421 35422 35423 35424 35425 35426 35427 35428 35429 35430 35431 35432 35433 35434 35435 35436 35437 35438 35439 35440 35441 35442 35443 35444 35445 35446 35447 35448 35449 35450 35451 35452 35453 35454 35455 35456 35457 35458 35459 35460 35461 35462 35463 35464 35465 35466 35467 35468 35469 35470 35471 35472 35473 35474 35475 35476 35477 35478 35479 35480 35481 35482 35483 35484 35485 35486 35487 35488 35489 35490 35491 35492 35493 35494 35495 35496 35497 35498 35499 35500 35501 35502 35503 35504 35505 35506 35507 35508 35509 35510 35511 35512 35513 35514 35515 35516 35517 35518 35519 35520 35521 35522 35523 35524 35525 35526 35527 35528 35529 35530 35531 35532 35533 35534 35535 35536 35537 35538 35539 35540 35541 35542 35543 35544 35545 35546 35547 35548 35549 35550 35551 35552 35553 35554 35555 35556 35557 35558 35559 35560 35561 35562 35563 35564 35565 35566 35567 35568 35569 35570 35571 35572 35573 35574 35575 35576 35577 35578 35579 35580 35581 35582 35583 35584 35585 35586 35587 35588 35589 35590 35591 35592 35593 35594 35595 35596 35597 35598 35599 35600 35601 35602 35603 35604 35605 35606 35607 35608 35609 35610 35611 35612 35613 35614 35615 35616 35617 35618 35619 35620 35621 35622 35623 35624 35625 35626 35627 35628 35629 35630 35631 35632 35633 35634 35635 35636 35637 35638 35639 35640 35641 35642 35643 35644 35645 35646 35647 35648 35649 35650 35651 35652 35653 35654 35655 35656 35657 35658 35659 35660 35661 35662 35663 35664 35665 35666 35667 35668 35669 35670 35671 35672 35673 35674 35675 35676 35677 35678 35679 35680 35681 35682 35683 35684 35685 35686 35687 35688 35689 35690 35691 35692 35693 35694 35695 35696 35697 35698 35699 35700 35701 35702 35703 35704 35705 35706 35707 35708 35709 35710 35711 35712 35713 35714 35715 35716 35717 35718 35719 35720 35721 35722 35723 35724 35725 35726 35727 35728 35729 35730 35731 35732 35733 35734 35735 35736 35737 35738 35739 35740 35741 35742 35743 35744 35745 35746 35747 35748 35749 35750 35751 35752 35753 35754 35755 35756 35757 35758 35759 35760 35761 35762 35763 35764 35765 35766 35767 35768 35769 35770 35771 35772 35773 35774 35775 35776 35777 35778 35779 35780 35781 35782 35783 35784 35785 35786 35787 35788 35789 35790 35791 35792 35793 35794 35795 35796 35797 35798 35799 35800 35801 35802 35803 35804 35805 35806 35807 35808 35809 35810 35811 35812 35813 35814 35815 35816 35817 35818 35819 35820 35821 35822 35823 35824 35825 35826 35827 35828 35829 35830 35831 35832 35833 35834 35835 35836 35837 35838 35839 35840 35841 35842 35843 35844 35845 35846 35847 35848 35849 35850 35851 35852 35853 35854 35855 35856 35857 35858 35859 35860 35861 35862 35863 35864 35865 35866 35867 35868 35869 35870 35871 35872 35873 35874 35875 35876 35877 35878 35879 35880 35881 35882 35883 35884 35885 35886 35887 35888 35889 35890 35891 35892 35893 35894 35895 35896 35897 35898 35899 35900 35901 35902 35903 35904 35905 35906 35907 35908 35909 35910 35911 35912 35913 35914 35915 35916 35917 35918 35919 35920 35921 35922 35923 35924 35925 35926 35927 35928 35929 35930 35931 35932 35933 35934 35935 35936 35937 35938 35939 35940 35941 35942 35943 35944 35945 35946 35947 35948 35949 35950 35951 35952 35953 35954 35955 35956 35957 35958 35959 35960 35961 35962 35963 35964 35965 35966 35967 35968 35969 35970 35971 35972 35973 35974 35975 35976 35977 35978 35979 35980 35981 35982 35983 35984 35985 35986 35987 35988 35989 35990 35991 35992 35993 35994 35995 35996 35997 35998 35999 36000 36001 36002 36003 36004 36005 36006 36007 36008 36009 36010 36011 36012 36013 36014 36015 36016 36017 36018 36019 36020 36021 36022 36023 36024 36025 36026 36027 36028 36029 36030 36031 36032 36033 36034 36035 36036 36037 36038 36039 36040 36041 36042 36043 36044 36045 36046 36047 36048 36049 36050 36051 36052 36053 36054 36055 36056 36057 36058 36059 36060 36061 36062 36063 36064 36065 36066 36067 36068 36069 36070 36071 36072 36073 36074 36075 36076 36077 36078 36079 36080 36081 36082 36083 36084 36085 36086 36087 36088 36089 36090 36091 36092 36093 36094 36095 36096 36097 36098 36099 36100 36101 36102 36103 36104 36105 36106 36107 36108 36109 36110 36111 36112 36113 36114 36115 36116 36117 36118 36119 36120 36121 36122 36123 36124 36125 36126 36127 36128 36129 36130 36131 36132 36133 36134 36135 36136 36137 36138 36139 36140 36141 36142 36143 36144 36145 36146 36147 36148 36149 36150 36151 36152 36153 36154 36155 36156 36157 36158 36159 36160 36161 36162 36163 36164 36165 36166 36167 36168 36169 36170 36171 36172 36173 36174 36175 36176 36177 36178 36179 36180 36181 36182 36183 36184 36185 36186 36187 36188 36189 36190 36191 36192 36193 36194 36195 36196 36197 36198 36199 36200 36201 36202 36203 36204 36205 36206 36207 36208 36209 36210 36211 36212 36213 36214 36215 36216 36217 36218 36219 36220 36221 36222 36223 36224 36225 36226 36227 36228 36229 36230 36231 36232 36233 36234 36235 36236 36237 36238 36239 36240 36241 36242 36243 36244 36245 36246 36247 36248 36249 36250 36251 36252 36253 36254 36255 36256 36257 36258 36259 36260 36261 36262 36263 36264 36265 36266 36267 36268 36269 36270 36271 36272 36273 36274 36275 36276 36277 36278 36279 36280 36281 36282 36283 36284 36285 36286 36287 36288 36289 36290 36291 36292 36293 36294 36295 36296 36297 36298 36299 36300 36301 36302 36303 36304 36305 36306 36307 36308 36309 36310 36311 36312 36313 36314 36315 36316 36317 36318 36319 36320 36321 36322 36323 36324 36325 36326 36327 36328 36329 36330 36331 36332 36333 36334 36335 36336 36337 36338 36339 36340 36341 36342 36343 36344 36345 36346 36347 36348 36349 36350 36351 36352 36353 36354 36355 36356 36357 36358 36359 36360 36361 36362 36363 36364 36365 36366 36367 36368 36369 36370 36371 36372 36373 36374 36375 36376 36377 36378 36379 36380 36381 36382 36383 36384 36385 36386 36387 36388 36389 36390 36391 36392 36393 36394 36395 36396 36397 36398 36399 36400 36401 36402 36403 36404 36405 36406 36407 36408 36409 36410 36411 36412 36413 36414 36415 36416 36417 36418 36419 36420 36421 36422 36423 36424 36425 36426 36427 36428 36429 36430 36431 36432 36433 36434 36435 36436 36437 36438 36439 36440 36441 36442 36443 36444 36445 36446 36447 36448 36449 36450 36451 36452 36453 36454 36455 36456 36457 36458 36459 36460 36461 36462 36463 36464 36465 36466 36467 36468 36469 36470 36471 36472 36473 36474 36475 36476 36477 36478 36479 36480 36481 36482 36483 36484 36485 36486 36487 36488 36489 36490 36491 36492 36493 36494 36495 36496 36497 36498 36499 36500 36501 36502 36503 36504 36505 36506 36507 36508 36509 36510 36511 36512 36513 36514 36515 36516 36517 36518 36519 36520 36521 36522 36523 36524 36525 36526 36527 36528 36529 36530 36531 36532 36533 36534 36535 36536 36537 36538 36539 36540 36541 36542 36543 36544 36545 36546 36547 36548 36549 36550 36551 36552 36553 36554 36555 36556 36557 36558 36559 36560 36561 36562 36563 36564 36565 36566 36567 36568 36569 36570 36571 36572 36573 36574 36575 36576 36577 36578 36579 36580 36581 36582 36583 36584 36585 36586 36587 36588 36589 36590 36591 36592 36593 36594 36595 36596 36597 36598 36599 36600 36601 36602 36603 36604 36605 36606 36607 36608 36609 36610 36611 36612 36613 36614 36615 36616 36617 36618 36619 36620 36621 36622 36623 36624 36625 36626 36627 36628 36629 36630 36631 36632 36633 36634 36635 36636 36637 36638 36639 36640 36641 36642 36643 36644 36645 36646 36647 36648 36649 36650 36651 36652 36653 36654 36655 36656 36657 36658 36659 36660 36661 36662 36663 36664 36665 36666 36667 36668 36669 36670 36671 36672 36673 36674 36675 36676 36677 36678 36679 36680 36681 36682 36683 36684 36685 36686 36687 36688 36689 36690 36691 36692 36693 36694 36695 36696 36697 36698 36699 36700 36701 36702 36703 36704 36705 36706 36707 36708 36709 36710 36711 36712 36713 36714 36715 36716 36717 36718 36719 36720 36721 36722 36723 36724 36725 36726 36727 36728 36729 36730 36731 36732 36733 36734 36735 36736 36737 36738 36739 36740 36741 36742 36743 36744 36745 36746 36747 36748 36749 36750 36751 36752 36753 36754 36755 36756 36757 36758 36759 36760 36761 36762 36763 36764 36765 36766 36767 36768 36769 36770 36771 36772 36773 36774 36775 36776 36777 36778 36779 36780 36781 36782 36783 36784 36785 36786 36787 36788 36789 36790 36791 36792 36793 36794 36795 36796 36797 36798 36799 36800 36801 36802 36803 36804 36805 36806 36807 36808 36809 36810 36811 36812 36813 36814 36815 36816 36817 36818 36819 36820 36821 36822 36823 36824 36825 36826 36827 36828 36829 36830 36831 36832 36833 36834 36835 36836 36837 36838 36839 36840 36841 36842 36843 36844 36845 36846 36847 36848 36849 36850 36851 36852 36853 36854 36855 36856 36857 36858 36859 36860 36861 36862 36863 36864 36865 36866 36867 36868 36869 36870 36871 36872 36873 36874 36875 36876 36877 36878 36879 36880 36881 36882 36883 36884 36885 36886 36887 36888 36889 36890 36891 36892 36893 36894 36895 36896 36897 36898 36899 36900 36901 36902 36903 36904 36905 36906 36907 36908 36909 36910 36911 36912 36913 36914 36915 36916 36917 36918 36919 36920 36921 36922 36923 36924 36925 36926 36927 36928 36929 36930 36931 36932 36933 36934 36935 36936 36937 36938 36939 36940 36941 36942 36943 36944 36945 36946 36947 36948 36949 36950 36951 36952 36953 36954 36955 36956 36957 36958 36959 36960 36961 36962 36963 36964 36965 36966 36967 36968 36969 36970 36971 36972 36973 36974 36975 36976 36977 36978 36979 36980 36981 36982 36983 36984 36985 36986 36987 36988 36989 36990 36991 36992 36993 36994 36995 36996 36997 36998 36999 37000 37001 37002 37003 37004 37005 37006 37007 37008 37009 37010 37011 37012 37013 37014 37015 37016 37017 37018 37019 37020 37021 37022 37023 37024 37025 37026 37027 37028 37029 37030 37031 37032 37033 37034 37035 37036 37037 37038 37039 37040 37041 37042 37043 37044 37045 37046 37047 37048 37049 37050 37051 37052 37053 37054 37055 37056 37057 37058 37059 37060 37061 37062 37063 37064 37065 37066 37067 37068 37069 37070 37071 37072 37073 37074 37075 37076 37077 37078 37079 37080 37081 37082 37083 37084 37085 37086 37087 37088 37089 37090 37091 37092 37093 37094 37095 37096 37097 37098 37099 37100 37101 37102 37103 37104 37105 37106 37107 37108 37109 37110 37111 37112 37113 37114 37115 37116 37117 37118 37119 37120 37121 37122 37123 37124 37125 37126 37127 37128 37129 37130 37131 37132 37133 37134 37135 37136 37137 37138 37139 37140 37141 37142 37143 37144 37145 37146 37147 37148 37149 37150 37151 37152 37153 37154 37155 37156 37157 37158 37159 37160 37161 37162 37163 37164 37165 37166 37167 37168 37169 37170 37171 37172 37173 37174 37175 37176 37177 37178 37179 37180 37181 37182 37183 37184 37185 37186 37187 37188 37189 37190 37191 37192 37193 37194 37195 37196 37197 37198 37199 37200 37201 37202 37203 37204 37205 37206 37207 37208 37209 37210 37211 37212 37213 37214 37215 37216 37217 37218 37219 37220 37221 37222 37223 37224 37225 37226 37227 37228 37229 37230 37231 37232 37233 37234 37235 37236 37237 37238 37239 37240 37241 37242 37243 37244 37245 37246 37247 37248 37249 37250 37251 37252 37253 37254 37255 37256 37257 37258 37259 37260 37261 37262 37263 37264 37265 37266 37267 37268 37269 37270 37271 37272 37273 37274 37275 37276 37277 37278 37279 37280 37281 37282 37283 37284 37285 37286 37287 37288 37289 37290 37291 37292 37293 37294 37295 37296 37297 37298 37299 37300 37301 37302 37303 37304 37305 37306 37307 37308 37309 37310 37311 37312 37313 37314 37315 37316 37317 37318 37319 37320 37321 37322 37323 37324 37325 37326 37327 37328 37329 37330 37331 37332 37333 37334 37335 37336 37337 37338 37339 37340 37341 37342 37343 37344 37345 37346 37347 37348 37349 37350 37351 37352 37353 37354 37355 37356 37357 37358 37359 37360 37361 37362 37363 37364 37365 37366 37367 37368 37369 37370 37371 37372 37373 37374 37375 37376 37377 37378 37379 37380 37381 37382 37383 37384 37385 37386 37387 37388 37389 37390 37391 37392 37393 37394 37395 37396 37397 37398 37399 37400 37401 37402 37403 37404 37405 37406 37407 37408 37409 37410 37411 37412 37413 37414 37415 37416 37417 37418 37419 37420 37421 37422 37423 37424 37425 37426 37427 37428 37429 37430 37431 37432 37433 37434 37435 37436 37437 37438 37439 37440 37441 37442 37443 37444 37445 37446 37447 37448 37449 37450 37451 37452 37453 37454 37455 37456 37457 37458 37459 37460 37461 37462 37463 37464 37465 37466 37467 37468 37469 37470 37471 37472 37473 37474 37475 37476 37477 37478 37479 37480 37481 37482 37483 37484 37485 37486 37487 37488 37489 37490 37491 37492 37493 37494 37495 37496 37497 37498 37499 37500 37501 37502 37503 37504 37505 37506 37507 37508 37509 37510 37511 37512 37513 37514 37515 37516 37517 37518 37519 37520 37521 37522 37523 37524 37525 37526 37527 37528 37529 37530 37531 37532 37533 37534 37535 37536 37537 37538 37539 37540 37541 37542 37543 37544 37545 37546 37547 37548 37549 37550 37551 37552 37553 37554 37555 37556 37557 37558 37559 37560 37561 37562 37563 37564 37565 37566 37567 37568 37569 37570 37571 37572 37573 37574 37575 37576 37577 37578 37579 37580 37581 37582 37583 37584 37585 37586 37587 37588 37589 37590 37591 37592 37593 37594 37595 37596 37597 37598 37599 37600 37601 37602 37603 37604 37605 37606 37607 37608 37609 37610 37611 37612 37613 37614 37615 37616 37617 37618 37619 37620 37621 37622 37623 37624 37625 37626 37627 37628 37629 37630 37631 37632 37633 37634 37635 37636 37637 37638 37639 37640 37641 37642 37643 37644 37645 37646 37647 37648 37649 37650 37651 37652 37653 37654 37655 37656 37657 37658 37659 37660 37661 37662 37663 37664 37665 37666 37667 37668 37669 37670 37671 37672 37673 37674 37675 37676 37677 37678 37679 37680 37681 37682 37683 37684 37685 37686 37687 37688 37689 37690 37691 37692 37693 37694 37695 37696 37697 37698 37699 37700 37701 37702 37703 37704 37705 37706 37707 37708 37709 37710 37711 37712 37713 37714 37715 37716 37717 37718 37719 37720 37721 37722 37723 37724 37725 37726 37727 37728 37729 37730 37731 37732 37733 37734 37735 37736 37737 37738 37739 37740 37741 37742 37743 37744 37745 37746 37747 37748 37749 37750 37751 37752 37753 37754 37755 37756 37757 37758 37759 37760 37761 37762 37763 37764 37765 37766 37767 37768 37769 37770 37771 37772 37773 37774 37775 37776 37777 37778 37779 37780 37781 37782 37783 37784 37785 37786 37787 37788 37789 37790 37791 37792 37793 37794 37795 37796 37797 37798 37799 37800 37801 37802 37803 37804 37805 37806 37807 37808 37809 37810 37811 37812 37813 37814 37815 37816 37817 37818 37819 37820 37821 37822 37823 37824 37825 37826 37827 37828 37829 37830 37831 37832 37833 37834 37835 37836 37837 37838 37839 37840 37841 37842 37843 37844 37845 37846 37847 37848 37849 37850 37851 37852 37853 37854 37855 37856 37857 37858 37859 37860 37861 37862 37863 37864 37865 37866 37867 37868 37869 37870 37871 37872 37873 37874 37875 37876 37877 37878 37879 37880 37881 37882 37883 37884 37885 37886 37887 37888 37889 37890 37891 37892 37893 37894 37895 37896 37897 37898 37899 37900 37901 37902 37903 37904 37905 37906 37907 37908 37909 37910 37911 37912 37913 37914 37915 37916 37917 37918 37919 37920 37921 37922 37923 37924 37925 37926 37927 37928 37929 37930 37931 37932 37933 37934 37935 37936 37937 37938 37939 37940 37941 37942 37943 37944 37945 37946 37947 37948 37949 37950 37951 37952 37953 37954 37955 37956 37957 37958 37959 37960 37961 37962 37963 37964 37965 37966 37967 37968 37969 37970 37971 37972 37973 37974 37975 37976 37977 37978 37979 37980 37981 37982 37983 37984 37985 37986 37987 37988 37989 37990 37991 37992 37993 37994 37995 37996 37997 37998 37999 38000 38001 38002 38003 38004 38005 38006 38007 38008 38009 38010 38011 38012 38013 38014 38015 38016 38017 38018 38019 38020 38021 38022 38023 38024 38025 38026 38027 38028 38029 38030 38031 38032 38033 38034 38035 38036 38037 38038 38039 38040 38041 38042 38043 38044 38045 38046 38047 38048 38049 38050 38051 38052 38053 38054 38055 38056 38057 38058 38059 38060 38061 38062 38063 38064 38065 38066 38067 38068 38069 38070 38071 38072 38073 38074 38075 38076 38077 38078 38079 38080 38081 38082 38083 38084 38085 38086 38087 38088 38089 38090 38091 38092 38093 38094 38095 38096 38097 38098 38099 38100 38101 38102 38103 38104 38105 38106 38107 38108 38109 38110 38111 38112 38113 38114 38115 38116 38117 38118 38119 38120 38121 38122 38123 38124 38125 38126 38127 38128 38129 38130 38131 38132 38133 38134 38135 38136 38137 38138 38139 38140 38141 38142 38143 38144 38145 38146 38147 38148 38149 38150 38151 38152 38153 38154 38155 38156 38157 38158 38159 38160 38161 38162 38163 38164 38165 38166 38167 38168 38169 38170 38171 38172 38173 38174 38175 38176 38177 38178 38179 38180 38181 38182 38183 38184 38185 38186 38187 38188 38189 38190 38191 38192 38193 38194 38195 38196 38197 38198 38199 38200 38201 38202 38203 38204 38205 38206 38207 38208 38209 38210 38211 38212 38213 38214 38215 38216 38217 38218 38219 38220 38221 38222 38223 38224 38225 38226 38227 38228 38229 38230 38231 38232 38233 38234 38235 38236 38237 38238 38239 38240 38241 38242 38243 38244 38245 38246 38247 38248 38249 38250 38251 38252 38253 38254 38255 38256 38257 38258 38259 38260 38261 38262 38263 38264 38265 38266 38267 38268 38269 38270 38271 38272 38273 38274 38275 38276 38277 38278 38279 38280 38281 38282 38283 38284 38285 38286 38287 38288 38289 38290 38291 38292 38293 38294 38295 38296 38297 38298 38299 38300 38301 38302 38303 38304 38305 38306 38307 38308 38309 38310 38311 38312 38313 38314 38315 38316 38317 38318 38319 38320 38321 38322 38323 38324 38325 38326 38327 38328 38329 38330 38331 38332 38333 38334 38335 38336 38337 38338 38339 38340 38341 38342 38343 38344 38345 38346 38347 38348 38349 38350 38351 38352 38353 38354 38355 38356 38357 38358 38359 38360 38361 38362 38363 38364 38365 38366 38367 38368 38369 38370 38371 38372 38373 38374 38375 38376 38377 38378 38379 38380 38381 38382 38383 38384 38385 38386 38387 38388 38389 38390 38391 38392 38393 38394 38395 38396 38397 38398 38399 38400 38401 38402 38403 38404 38405 38406 38407 38408 38409 38410 38411 38412 38413 38414 38415 38416 38417 38418 38419 38420 38421 38422 38423 38424 38425 38426 38427 38428 38429 38430 38431 38432 38433 38434 38435 38436 38437 38438 38439 38440 38441 38442 38443 38444 38445 38446 38447 38448 38449 38450 38451 38452 38453 38454 38455 38456 38457 38458 38459 38460 38461 38462 38463 38464 38465 38466 38467 38468 38469 38470 38471 38472 38473 38474 38475 38476 38477 38478 38479 38480 38481 38482 38483 38484 38485 38486 38487 38488 38489 38490 38491 38492 38493 38494 38495 38496 38497 38498 38499 38500 38501 38502 38503 38504 38505 38506 38507 38508 38509 38510 38511 38512 38513 38514 38515 38516 38517 38518 38519 38520 38521 38522 38523 38524 38525 38526 38527 38528 38529 38530 38531 38532 38533 38534 38535 38536 38537 38538 38539 38540 38541 38542 38543 38544 38545 38546 38547 38548 38549 38550 38551 38552 38553 38554 38555 38556 38557 38558 38559 38560 38561 38562 38563 38564 38565 38566 38567 38568 38569 38570 38571 38572 38573 38574 38575 38576 38577 38578 38579 38580 38581 38582 38583 38584 38585 38586 38587 38588 38589 38590 38591 38592 38593 38594 38595 38596 38597 38598 38599 38600 38601 38602 38603 38604 38605 38606 38607 38608 38609 38610 38611 38612 38613 38614 38615 38616 38617 38618 38619 38620 38621 38622 38623 38624 38625 38626 38627 38628 38629 38630 38631 38632 38633 38634 38635 38636 38637 38638 38639 38640 38641 38642 38643 38644 38645 38646 38647 38648 38649 38650 38651 38652 38653 38654 38655 38656 38657 38658 38659 38660 38661 38662 38663 38664 38665 38666 38667 38668 38669 38670 38671 38672 38673 38674 38675 38676 38677 38678 38679 38680 38681 38682 38683 38684 38685 38686 38687 38688 38689 38690 38691 38692 38693 38694 38695 38696 38697 38698 38699 38700 38701 38702 38703 38704 38705 38706 38707 38708 38709 38710 38711 38712 38713 38714 38715 38716 38717 38718 38719 38720 38721 38722 38723 38724 38725 38726 38727 38728 38729 38730 38731 38732 38733 38734 38735 38736 38737 38738 38739 38740 38741 38742 38743 38744 38745 38746 38747 38748 38749 38750 38751 38752 38753 38754 38755 38756 38757 38758 38759 38760 38761 38762 38763 38764 38765 38766 38767 38768 38769 38770 38771 38772 38773 38774 38775 38776 38777 38778 38779 38780 38781 38782 38783 38784 38785 38786 38787 38788 38789 38790 38791 38792 38793 38794 38795 38796 38797 38798 38799 38800 38801 38802 38803 38804 38805 38806 38807 38808 38809 38810 38811 38812 38813 38814 38815 38816 38817 38818 38819 38820 38821 38822 38823 38824 38825 38826 38827 38828 38829 38830 38831 38832 38833 38834 38835 38836 38837 38838 38839 38840 38841 38842 38843 38844 38845 38846 38847 38848 38849 38850 38851 38852 38853 38854 38855 38856 38857 38858 38859 38860 38861 38862 38863 38864 38865 38866 38867 38868 38869 38870 38871 38872 38873 38874 38875 38876 38877 38878 38879 38880 38881 38882 38883 38884 38885 38886 38887 38888 38889 38890 38891 38892 38893 38894 38895 38896 38897 38898 38899 38900 38901 38902 38903 38904 38905 38906 38907 38908 38909 38910 38911 38912 38913 38914 38915 38916 38917 38918 38919 38920 38921 38922 38923 38924 38925 38926 38927 38928 38929 38930 38931 38932 38933 38934 38935 38936 38937 38938 38939 38940 38941 38942 38943 38944 38945 38946 38947 38948 38949 38950 38951 38952 38953 38954 38955 38956 38957 38958 38959 38960 38961 38962 38963 38964 38965 38966 38967 38968 38969 38970 38971 38972 38973 38974 38975 38976 38977 38978 38979 38980 38981 38982 38983 38984 38985 38986 38987 38988 38989 38990 38991 38992 38993 38994 38995 38996 38997 38998 38999 39000 39001 39002 39003 39004 39005 39006 39007 39008 39009 39010 39011 39012 39013 39014 39015 39016 39017 39018 39019 39020 39021 39022 39023 39024 39025 39026 39027 39028 39029 39030 39031 39032 39033 39034 39035 39036 39037 39038 39039 39040 39041 39042 39043 39044 39045 39046 39047 39048 39049 39050 39051 39052 39053 39054 39055 39056 39057 39058 39059 39060 39061 39062 39063 39064 39065 39066 39067 39068 39069 39070 39071 39072 39073 39074 39075 39076 39077 39078 39079 39080 39081 39082 39083 39084 39085 39086 39087 39088 39089 39090 39091 39092 39093 39094 39095 39096 39097 39098 39099 39100 39101 39102 39103 39104 39105 39106 39107 39108 39109 39110 39111 39112 39113 39114 39115 39116 39117 39118 39119 39120 39121 39122 39123 39124 39125 39126 39127 39128 39129 39130 39131 39132 39133 39134 39135 39136 39137 39138 39139 39140 39141 39142 39143 39144 39145 39146 39147 39148 39149 39150 39151 39152 39153 39154 39155 39156 39157 39158 39159 39160 39161 39162 39163 39164 39165 39166 39167 39168 39169 39170 39171 39172 39173 39174 39175 39176 39177 39178 39179 39180 39181 39182 39183 39184 39185 39186 39187 39188 39189 39190 39191 39192 39193 39194 39195 39196 39197 39198 39199 39200 39201 39202 39203 39204 39205 39206 39207 39208 39209 39210 39211 39212 39213 39214 39215 39216 39217 39218 39219 39220 39221 39222 39223 39224 39225 39226 39227 39228 39229 39230 39231 39232 39233 39234 39235 39236 39237 39238 39239 39240 39241 39242 39243 39244 39245 39246 39247 39248 39249 39250 39251 39252 39253 39254 39255 39256 39257 39258 39259 39260 39261 39262 39263 39264 39265 39266 39267 39268 39269 39270 39271 39272 39273 39274 39275 39276 39277 39278 39279 39280 39281 39282 39283 39284 39285 39286 39287 39288 39289 39290 39291 39292 39293 39294 39295 39296 39297 39298 39299 39300 39301 39302 39303 39304 39305 39306 39307 39308 39309 39310 39311 39312 39313 39314 39315 39316 39317 39318 39319 39320 39321 39322 39323 39324 39325 39326 39327 39328 39329 39330 39331 39332 39333 39334 39335 39336 39337 39338 39339 39340 39341 39342 39343 39344 39345 39346 39347 39348 39349 39350 39351 39352 39353 39354 39355 39356 39357 39358 39359 39360 39361 39362 39363 39364 39365 39366 39367 39368 39369 39370 39371 39372 39373 39374 39375 39376 39377 39378 39379 39380 39381 39382 39383 39384 39385 39386 39387 39388 39389 39390 39391 39392 39393 39394 39395 39396 39397 39398 39399 39400 39401 39402 39403 39404 39405 39406 39407 39408 39409 39410 39411 39412 39413 39414 39415 39416 39417 39418 39419 39420 39421 39422 39423 39424 39425 39426 39427 39428 39429 39430 39431 39432 39433 39434 39435 39436 39437 39438 39439 39440 39441 39442 39443 39444 39445 39446 39447 39448 39449 39450 39451 39452 39453 39454 39455 39456 39457 39458 39459 39460 39461 39462 39463 39464 39465 39466 39467 39468 39469 39470 39471 39472 39473 39474 39475 39476 39477 39478 39479 39480 39481 39482 39483 39484 39485 39486 39487 39488 39489 39490 39491 39492 39493 39494 39495 39496 39497 39498 39499 39500 39501 39502 39503 39504 39505 39506 39507 39508 39509 39510 39511 39512 39513 39514 39515 39516 39517 39518 39519 39520 39521 39522 39523 39524 39525 39526 39527 39528 39529 39530 39531 39532 39533 39534 39535 39536 39537 39538 39539 39540 39541 39542 39543 39544 39545 39546 39547 39548 39549 39550 39551 39552 39553 39554 39555 39556 39557 39558 39559 39560 39561 39562 39563 39564 39565 39566 39567 39568 39569 39570 39571 39572 39573 39574 39575 39576 39577 39578 39579 39580 39581 39582 39583 39584 39585 39586 39587 39588 39589 39590 39591 39592 39593 39594 39595 39596 39597 39598 39599 39600 39601 39602 39603 39604 39605 39606 39607 39608 39609 39610 39611 39612 39613 39614 39615 39616 39617 39618 39619 39620 39621 39622 39623 39624 39625 39626 39627 39628 39629 39630 39631 39632 39633 39634 39635 39636 39637 39638 39639 39640 39641 39642 39643 39644 39645 39646 39647 39648 39649 39650 39651 39652 39653 39654 39655 39656 39657 39658 39659 39660 39661 39662 39663 39664 39665 39666 39667 39668 39669 39670 39671 39672 39673 39674 39675 39676 39677 39678 39679 39680 39681 39682 39683 39684 39685 39686 39687 39688 39689 39690 39691 39692 39693 39694 39695 39696 39697 39698 39699 39700 39701 39702 39703 39704 39705 39706 39707 39708 39709 39710 39711 39712 39713 39714 39715 39716 39717 39718 39719 39720 39721 39722 39723 39724 39725 39726 39727 39728 39729 39730 39731 39732 39733 39734 39735 39736 39737 39738 39739 39740 39741 39742 39743 39744 39745 39746 39747 39748 39749 39750 39751 39752 39753 39754 39755 39756 39757 39758 39759 39760 39761 39762 39763 39764 39765 39766 39767 39768 39769 39770 39771 39772 39773 39774 39775 39776 39777 39778 39779 39780 39781 39782 39783 39784 39785 39786 39787 39788 39789 39790 39791 39792 39793 39794 39795 39796 39797 39798 39799 39800 39801 39802 39803 39804 39805 39806 39807 39808 39809 39810 39811 39812 39813 39814 39815 39816 39817 39818 39819 39820 39821 39822 39823 39824 39825 39826 39827 39828 39829 39830 39831 39832 39833 39834 39835 39836 39837 39838 39839 39840 39841 39842 39843 39844 39845 39846 39847 39848 39849 39850 39851 39852 39853 39854 39855 39856 39857 39858 39859 39860 39861 39862 39863 39864 39865 39866 39867 39868 39869 39870 39871 39872 39873 39874 39875 39876 39877 39878 39879 39880 39881 39882 39883 39884 39885 39886 39887 39888 39889 39890 39891 39892 39893 39894 39895 39896 39897 39898 39899 39900 39901 39902 39903 39904 39905 39906 39907 39908 39909 39910 39911 39912 39913 39914 39915 39916 39917 39918 39919 39920 39921 39922 39923 39924 39925 39926 39927 39928 39929 39930 39931 39932 39933 39934 39935 39936 39937 39938 39939 39940 39941 39942 39943 39944 39945 39946 39947 39948 39949 39950 39951 39952 39953 39954 39955 39956 39957 39958 39959 39960 39961 39962 39963 39964 39965 39966 39967 39968 39969 39970 39971 39972 39973 39974 39975 39976 39977 39978 39979 39980 39981 39982 39983 39984 39985 39986 39987 39988 39989 39990 39991 39992 39993 39994 39995 39996 39997 39998 39999 40000 40001 40002 40003 40004 40005 40006 40007 40008 40009 40010 40011 40012 40013 40014 40015 40016 40017 40018 40019 40020 40021 40022 40023 40024 40025 40026 40027 40028 40029 40030 40031 40032 40033 40034 40035 40036 40037 40038 40039 40040 40041 40042 40043 40044 40045 40046 40047 40048 40049 40050 40051 40052 40053 40054 40055 40056 40057 40058 40059 40060 40061 40062 40063 40064 40065 40066 40067 40068 40069 40070 40071 40072 40073 40074 40075 40076 40077 40078 40079 40080 40081 40082 40083 40084 40085 40086 40087 40088 40089 40090 40091 40092 40093 40094 40095 40096 40097 40098 40099 40100 40101 40102 40103 40104 40105 40106 40107 40108 40109 40110 40111 40112 40113 40114 40115 40116 40117 40118 40119 40120 40121 40122 40123 40124 40125 40126 40127 40128 40129 40130 40131 40132 40133 40134 40135 40136 40137 40138 40139 40140 40141 40142 40143 40144 40145 40146 40147 40148 40149 40150 40151 40152 40153 40154 40155 40156 40157 40158 40159 40160 40161 40162 40163 40164 40165 40166 40167 40168 40169 40170 40171 40172 40173 40174 40175 40176 40177 40178 40179 40180 40181 40182 40183 40184 40185 40186 40187 40188 40189 40190 40191 40192 40193 40194 40195 40196 40197 40198 40199 40200 40201 40202 40203 40204 40205 40206 40207 40208 40209 40210 40211 40212 40213 40214 40215 40216 40217 40218 40219 40220 40221 40222 40223 40224 40225 40226 40227 40228 40229 40230 40231 40232 40233 40234 40235 40236 40237 40238 40239 40240 40241 40242 40243 40244 40245 40246 40247 40248 40249 40250 40251 40252 40253 40254 40255 40256 40257 40258 40259 40260 40261 40262 40263 40264 40265 40266 40267 40268 40269 40270 40271 40272 40273 40274 40275 40276 40277 40278 40279 40280 40281 40282 40283 40284 40285 40286 40287 40288 40289 40290 40291 40292 40293 40294 40295 40296 40297 40298 40299 40300 40301 40302 40303 40304 40305 40306 40307 40308 40309 40310 40311 40312 40313 40314 40315 40316 40317 40318 40319 40320 40321 40322 40323 40324 40325 40326 40327 40328 40329 40330 40331 40332 40333 40334 40335 40336 40337 40338 40339 40340 40341 40342 40343 40344 40345 40346 40347 40348 40349 40350 40351 40352 40353 40354 40355 40356 40357 40358 40359 40360 40361 40362 40363 40364 40365 40366 40367 40368 40369 40370 40371 40372 40373 40374 40375 40376 40377 40378 40379 40380 40381 40382 40383 40384 40385 40386 40387 40388 40389 40390 40391 40392 40393 40394 40395 40396 40397 40398 40399 40400 40401 40402 40403 40404 40405 40406 40407 40408 40409 40410 40411 40412 40413 40414 40415 40416 40417 40418 40419 40420 40421 40422 40423 40424 40425 40426 40427 40428 40429 40430 40431 40432 40433 40434 40435 40436 40437 40438 40439 40440 40441 40442 40443 40444 40445 40446 40447 40448 40449 40450 40451 40452 40453 40454 40455 40456 40457 40458 40459 40460 40461 40462 40463 40464 40465 40466 40467 40468 40469 40470 40471 40472 40473 40474 40475 40476 40477 40478 40479 40480 40481 40482 40483 40484 40485 40486 40487 40488 40489 40490 40491 40492 40493 40494 40495 40496 40497 40498 40499 40500 40501 40502 40503 40504 40505 40506 40507 40508 40509 40510 40511 40512 40513 40514 40515 40516 40517 40518 40519 40520 40521 40522 40523 40524 40525 40526 40527 40528 40529 40530 40531 40532 40533 40534 40535 40536 40537 40538 40539 40540 40541 40542 40543 40544 40545 40546 40547 40548 40549 40550 40551 40552 40553 40554 40555 40556 40557 40558 40559 40560 40561 40562 40563 40564 40565 40566 40567 40568 40569 40570 40571 40572 40573 40574 40575 40576 40577 40578 40579 40580 40581 40582 40583 40584 40585 40586 40587 40588 40589 40590 40591 40592 40593 40594 40595 40596 40597 40598 40599 40600 40601 40602 40603 40604 40605 40606 40607 40608 40609 40610 40611 40612 40613 40614 40615 40616 40617 40618 40619 40620 40621 40622 40623 40624 40625 40626 40627 40628 40629 40630 40631 40632 40633 40634 40635 40636 40637 40638 40639 40640 40641 40642 40643 40644 40645 40646 40647 40648 40649 40650 40651 40652 40653 40654 40655 40656 40657 40658 40659 40660 40661 40662 40663 40664 40665 40666 40667 40668 40669 40670 40671 40672 40673 40674 40675 40676 40677 40678 40679 40680 40681 40682 40683 40684 40685 40686 40687 40688 40689 40690 40691 40692 40693 40694 40695 40696 40697 40698 40699 40700 40701 40702 40703 40704 40705 40706 40707 40708 40709 40710 40711 40712 40713 40714 40715 40716 40717 40718 40719 40720 40721 40722 40723 40724 40725 40726 40727 40728 40729 40730 40731 40732 40733 40734 40735 40736 40737 40738 40739 40740 40741 40742 40743 40744 40745 40746 40747 40748 40749 40750 40751 40752 40753 40754 40755 40756 40757 40758 40759 40760 40761 40762 40763 40764 40765 40766 40767 40768 40769 40770 40771 40772 40773 40774 40775 40776 40777 40778 40779 40780 40781 40782 40783 40784 40785 40786 40787 40788 40789 40790 40791 40792 40793 40794 40795 40796 40797 40798 40799 40800 40801 40802 40803 40804 40805 40806 40807 40808 40809 40810 40811 40812 40813 40814 40815 40816 40817 40818 40819 40820 40821 40822 40823 40824 40825 40826 40827 40828 40829 40830 40831 40832 40833 40834 40835 40836 40837 40838 40839 40840 40841 40842 40843 40844 40845 40846 40847 40848 40849 40850 40851 40852 40853 40854 40855 40856 40857 40858 40859 40860 40861 40862 40863 40864 40865 40866 40867 40868 40869 40870 40871 40872 40873 40874 40875 40876 40877 40878 40879 40880 40881 40882 40883 40884 40885 40886 40887 40888 40889 40890 40891 40892 40893 40894 40895 40896 40897 40898 40899 40900 40901 40902 40903 40904 40905 40906 40907 40908 40909 40910 40911 40912 40913 40914 40915 40916 40917 40918 40919 40920 40921 40922 40923 40924 40925 40926 40927 40928 40929 40930 40931 40932 40933 40934 40935 40936 40937 40938 40939 40940 40941 40942 40943 40944 40945 40946 40947 40948 40949 40950 40951 40952 40953 40954 40955 40956 40957 40958 40959 40960 40961 40962 40963 40964 40965 40966 40967 40968 40969 40970 40971 40972 40973 40974 40975 40976 40977 40978 40979 40980 40981 40982 40983 40984 40985 40986 40987 40988 40989 40990 40991 40992 40993 40994 40995 40996 40997 40998 40999 41000 41001 41002 41003 41004 41005 41006 41007 41008 41009 41010 41011 41012 41013 41014 41015 41016 41017 41018 41019 41020 41021 41022 41023 41024 41025 41026 41027 41028 41029 41030 41031 41032 41033 41034 41035 41036 41037 41038 41039 41040 41041 41042 41043 41044 41045 41046 41047 41048 41049 41050 41051 41052 41053 41054 41055 41056 41057 41058 41059 41060 41061 41062 41063 41064 41065 41066 41067 41068 41069 41070 41071 41072 41073 41074 41075 41076 41077 41078 41079 41080 41081 41082 41083 41084 41085 41086 41087 41088 41089 41090 41091 41092 41093 41094 41095 41096 41097 41098 41099 41100 41101 41102 41103 41104 41105 41106 41107 41108 41109 41110 41111 41112 41113 41114 41115 41116 41117 41118 41119 41120 41121 41122 41123 41124 41125 41126 41127 41128 41129 41130 41131 41132 41133 41134 41135 41136 41137 41138 41139 41140 41141 41142 41143 41144 41145 41146 41147 41148 41149 41150 41151 41152 41153 41154 41155 41156 41157 41158 41159 41160 41161 41162 41163 41164 41165 41166 41167 41168 41169 41170 41171 41172 41173 41174 41175 41176 41177 41178 41179 41180 41181 41182 41183 41184 41185 41186 41187 41188 41189 41190 41191 41192 41193 41194 41195 41196 41197 41198 41199 41200 41201 41202 41203 41204 41205 41206 41207 41208 41209 41210 41211 41212 41213 41214 41215 41216 41217 41218 41219 41220 41221 41222 41223 41224 41225 41226 41227 41228 41229 41230 41231 41232 41233 41234 41235 41236 41237 41238 41239 41240 41241 41242 41243 41244 41245 41246 41247 41248 41249 41250 41251 41252 41253 41254 41255 41256 41257 41258 41259 41260 41261 41262 41263 41264 41265 41266 41267 41268 41269 41270 41271 41272 41273 41274 41275 41276 41277 41278 41279 41280 41281 41282 41283 41284 41285 41286 41287 41288 41289 41290 41291 41292 41293 41294 41295 41296 41297 41298 41299 41300 41301 41302 41303 41304 41305 41306 41307 41308 41309 41310 41311 41312 41313 41314 41315 41316 41317 41318 41319 41320 41321 41322 41323 41324 41325 41326 41327 41328 41329 41330 41331 41332 41333 41334 41335 41336 41337 41338 41339 41340 41341 41342 41343 41344 41345 41346 41347 41348 41349 41350 41351 41352 41353 41354 41355 41356 41357 41358 41359 41360 41361 41362 41363 41364 41365 41366 41367 41368 41369 41370 41371 41372 41373 41374 41375 41376 41377 41378 41379 41380 41381 41382 41383 41384 41385 41386 41387 41388 41389 41390 41391 41392 41393 41394 41395 41396 41397 41398 41399 41400 41401 41402 41403 41404 41405 41406 41407 41408 41409 41410 41411 41412 41413 41414 41415 41416 41417 41418 41419 41420 41421 41422 41423 41424 41425 41426 41427 41428 41429 41430 41431 41432 41433 41434 41435 41436 41437 41438 41439 41440 41441 41442 41443 41444 41445 41446 41447 41448 41449 41450 41451 41452 41453 41454 41455 41456 41457 41458 41459 41460 41461 41462 41463 41464 41465 41466 41467 41468 41469 41470 41471 41472 41473 41474 41475 41476 41477 41478 41479 41480 41481 41482 41483 41484 41485 41486 41487 41488 41489 41490 41491 41492 41493 41494 41495 41496 41497 41498 41499 41500 41501 41502 41503 41504 41505 41506 41507 41508 41509 41510 41511 41512 41513 41514 41515 41516 41517 41518 41519 41520 41521 41522 41523 41524 41525 41526 41527 41528 41529 41530 41531 41532 41533 41534 41535 41536 41537 41538 41539 41540 41541 41542 41543 41544 41545 41546 41547 41548 41549 41550 41551 41552 41553 41554 41555 41556 41557 41558 41559 41560 41561 41562 41563 41564 41565 41566 41567 41568 41569 41570 41571 41572 41573 41574 41575 41576 41577 41578 41579 41580 41581 41582 41583 41584 41585 41586 41587 41588 41589 41590 41591 41592 41593 41594 41595 41596 41597 41598 41599 41600 41601 41602 41603 41604 41605 41606 41607 41608 41609 41610 41611 41612 41613 41614 41615 41616 41617 41618 41619 41620 41621 41622 41623 41624 41625 41626 41627 41628 41629 41630 41631 41632 41633 41634 41635 41636 41637 41638 41639 41640 41641 41642 41643 41644 41645 41646 41647 41648 41649 41650 41651 41652 41653 41654 41655 41656 41657 41658 41659 41660 41661 41662 41663 41664 41665 41666 41667 41668 41669 41670 41671 41672 41673 41674 41675 41676 41677 41678 41679 41680 41681 41682 41683 41684 41685 41686 41687 41688 41689 41690 41691 41692 41693 41694 41695 41696 41697 41698 41699 41700 41701 41702 41703 41704 41705 41706 41707 41708 41709 41710 41711 41712 41713 41714 41715 41716 41717 41718 41719 41720 41721 41722 41723 41724 41725 41726 41727 41728 41729 41730 41731 41732 41733 41734 41735 41736 41737 41738 41739 41740 41741 41742 41743 41744 41745 41746 41747 41748 41749 41750 41751 41752 41753 41754 41755 41756 41757 41758 41759 41760 41761 41762 41763 41764 41765 41766 41767 41768 41769 41770 41771 41772 41773 41774 41775 41776 41777 41778 41779 41780 41781 41782 41783 41784 41785 41786 41787 41788 41789 41790 41791 41792 41793 41794 41795 41796 41797 41798 41799 41800 41801 41802 41803 41804 41805 41806 41807 41808 41809 41810 41811 41812 41813 41814 41815 41816 41817 41818 41819 41820 41821 41822 41823 41824 41825 41826 41827 41828 41829 41830 41831 41832 41833 41834 41835 41836 41837 41838 41839 41840 41841 41842 41843 41844 41845 41846 41847 41848 41849 41850 41851 41852 41853 41854 41855 41856 41857 41858 41859 41860 41861 41862 41863 41864 41865 41866 41867 41868 41869 41870 41871 41872 41873 41874 41875 41876 41877 41878 41879 41880 41881 41882 41883 41884 41885 41886 41887 41888 41889 41890 41891 41892 41893 41894 41895 41896 41897 41898 41899 41900 41901 41902 41903 41904 41905 41906 41907 41908 41909 41910 41911 41912 41913 41914 41915 41916 41917 41918 41919 41920 41921 41922 41923 41924 41925 41926 41927 41928 41929 41930 41931 41932 41933 41934 41935 41936 41937 41938 41939 41940 41941 41942 41943 41944 41945 41946 41947 41948 41949 41950 41951 41952 41953 41954 41955 41956 41957 41958 41959 41960 41961 41962 41963 41964 41965 41966 41967 41968 41969 41970 41971 41972 41973 41974 41975 41976 41977 41978 41979 41980 41981 41982 41983 41984 41985 41986 41987 41988 41989 41990 41991 41992 41993 41994 41995 41996 41997 41998 41999 42000 42001 42002 42003 42004 42005 42006 42007 42008 42009 42010 42011 42012 42013 42014 42015 42016 42017 42018 42019 42020 42021 42022 42023 42024 42025 42026 42027 42028 42029 42030 42031 42032 42033 42034 42035 42036 42037 42038 42039 42040 42041 42042 42043 42044 42045 42046 42047 42048 42049 42050 42051 42052 42053 42054 42055 42056 42057 42058 42059 42060 42061 42062 42063 42064 42065 42066 42067 42068 42069 42070 42071 42072 42073 42074 42075 42076 42077 42078 42079 42080 42081 42082 42083 42084 42085 42086 42087 42088 42089 42090 42091 42092 42093 42094 42095 42096 42097 42098 42099 42100 42101 42102 42103 42104 42105 42106 42107 42108 42109 42110 42111 42112 42113 42114 42115 42116 42117 42118 42119 42120 42121 42122 42123 42124 42125 42126 42127 42128 42129 42130 42131 42132 42133 42134 42135 42136 42137 42138 42139 42140 42141 42142 42143 42144 42145 42146 42147 42148 42149 42150 42151 42152 42153 42154 42155 42156 42157 42158 42159 42160 42161 42162 42163 42164 42165 42166 42167 42168 42169 42170 42171 42172 42173 42174 42175 42176 42177 42178 42179 42180 42181 42182 42183 42184 42185 42186 42187 42188 42189 42190 42191 42192 42193 42194 42195 42196 42197 42198 42199 42200 42201 42202 42203 42204 42205 42206 42207 42208 42209 42210 42211 42212 42213 42214 42215 42216 42217 42218 42219 42220 42221 42222 42223 42224 42225 42226 42227 42228 42229 42230 42231 42232 42233 42234 42235 42236 42237 42238 42239 42240 42241 42242 42243 42244 42245 42246 42247 42248 42249 42250 42251 42252 42253 42254 42255 42256 42257 42258 42259 42260 42261 42262 42263 42264 42265 42266 42267 42268 42269 42270 42271 42272 42273 42274 42275 42276 42277 42278 42279 42280 42281 42282 42283 42284 42285 42286 42287 42288 42289 42290 42291 42292 42293 42294 42295 42296 42297 42298 42299 42300 42301 42302 42303 42304 42305 42306 42307 42308 42309 42310 42311 42312 42313 42314 42315 42316 42317 42318 42319 42320 42321 42322 42323 42324 42325 42326 42327 42328 42329 42330 42331 42332 42333 42334 42335 42336 42337 42338 42339 42340 42341 42342 42343 42344 42345 42346 42347 42348 42349 42350 42351 42352 42353 42354 42355 42356 42357 42358 42359 42360 42361 42362 42363 42364 42365 42366 42367 42368 42369 42370 42371 42372 42373 42374 42375 42376 42377 42378 42379 42380 42381 42382 42383 42384 42385 42386 42387 42388 42389 42390 42391 42392 42393 42394 42395 42396 42397 42398 42399 42400 42401 42402 42403 42404 42405 42406 42407 42408 42409 42410 42411 42412 42413 42414 42415 42416 42417 42418 42419 42420 42421 42422 42423 42424 42425 42426 42427 42428 42429 42430 42431 42432 42433 42434 42435 42436 42437 42438 42439 42440 42441 42442 42443 42444 42445 42446 42447 42448 42449 42450 42451 42452 42453 42454 42455 42456 42457 42458 42459 42460 42461 42462 42463 42464 42465 42466 42467 42468 42469 42470 42471 42472 42473 42474 42475 42476 42477 42478 42479 42480 42481 42482 42483 42484 42485 42486 42487 42488 42489 42490 42491 42492 42493 42494 42495 42496 42497 42498 42499 42500 42501 42502 42503 42504 42505 42506 42507 42508 42509 42510 42511 42512 42513 42514 42515 42516 42517 42518 42519 42520 42521 42522 42523 42524 42525 42526 42527 42528 42529 42530 42531 42532 42533 42534 42535 42536 42537 42538 42539 42540 42541 42542 42543 42544 42545 42546 42547 42548 42549 42550 42551 42552 42553 42554 42555 42556 42557 42558 42559 42560 42561 42562 42563 42564 42565 42566 42567 42568 42569 42570 42571 42572 42573 42574 42575 42576 42577 42578 42579 42580 42581 42582 42583 42584 42585 42586 42587 42588 42589 42590 42591 42592 42593 42594 42595 42596 42597 42598 42599 42600 42601 42602 42603 42604 42605 42606 42607 42608 42609 42610 42611 42612 42613 42614 42615 42616 42617 42618 42619 42620 42621 42622 42623 42624 42625 42626 42627 42628 42629 42630 42631 42632 42633 42634 42635 42636 42637 42638 42639 42640 42641 42642 42643 42644 42645 42646 42647 42648 42649 42650 42651 42652 42653 42654 42655 42656 42657 42658 42659 42660 42661 42662 42663 42664 42665 42666 42667 42668 42669 42670 42671 42672 42673 42674 42675 42676 42677 42678 42679 42680 42681 42682 42683 42684 42685 42686 42687 42688 42689 42690 42691 42692 42693 42694 42695 42696 42697 42698 42699 42700 42701 42702 42703 42704 42705 42706 42707 42708 42709 42710 42711 42712 42713 42714 42715 42716 42717 42718 42719 42720 42721 42722 42723 42724 42725 42726 42727 42728 42729 42730 42731 42732 42733 42734 42735 42736 42737 42738 42739 42740 42741 42742 42743 42744 42745 42746 42747 42748 42749 42750 42751 42752 42753 42754 42755 42756 42757 42758 42759 42760 42761 42762 42763 42764 42765 42766 42767 42768 42769 42770 42771 42772 42773 42774 42775 42776 42777 42778 42779 42780 42781 42782 42783 42784 42785 42786 42787 42788 42789 42790 42791 42792 42793 42794 42795 42796 42797 42798 42799 42800 42801 42802 42803 42804 42805 42806 42807 42808 42809 42810 42811 42812 42813 42814 42815 42816 42817 42818 42819 42820 42821 42822 42823 42824 42825 42826 42827 42828 42829 42830 42831 42832 42833 42834 42835 42836 42837 42838 42839 42840 42841 42842 42843 42844 42845 42846 42847 42848 42849 42850 42851 42852 42853 42854 42855 42856 42857 42858 42859 42860 42861 42862 42863 42864 42865 42866 42867 42868 42869 42870 42871 42872 42873 42874 42875 42876 42877 42878 42879 42880 42881 42882 42883 42884 42885 42886 42887 42888 42889 42890 42891 42892 42893 42894 42895 42896 42897 42898 42899 42900 42901 42902 42903 42904 42905 42906 42907 42908 42909 42910 42911 42912 42913 42914 42915 42916 42917 42918 42919 42920 42921 42922 42923 42924 42925 42926 42927 42928 42929 42930 42931 42932 42933 42934 42935 42936 42937 42938 42939 42940 42941 42942 42943 42944 42945 42946 42947 42948 42949 42950 42951 42952 42953 42954 42955 42956 42957 42958 42959 42960 42961 42962 42963 42964 42965 42966 42967 42968 42969 42970 42971 42972 42973 42974 42975 42976 42977 42978 42979 42980 42981 42982 42983 42984 42985 42986 42987 42988 42989 42990 42991 42992 42993 42994 42995 42996 42997 42998 42999 43000 43001 43002 43003 43004 43005 43006 43007 43008 43009 43010 43011 43012 43013 43014 43015 43016 43017 43018 43019 43020 43021 43022 43023 43024 43025 43026 43027 43028 43029 43030 43031 43032 43033 43034 43035 43036 43037 43038 43039 43040 43041 43042 43043 43044 43045 43046 43047 43048 43049 43050 43051 43052 43053 43054 43055 43056 43057 43058 43059 43060 43061 43062 43063 43064 43065 43066 43067 43068 43069 43070 43071 43072 43073 43074 43075 43076 43077 43078 43079 43080 43081 43082 43083 43084 43085 43086 43087 43088 43089 43090 43091 43092 43093 43094 43095 43096 43097 43098 43099 43100 43101 43102 43103 43104 43105 43106 43107 43108 43109 43110 43111 43112 43113 43114 43115 43116 43117 43118 43119 43120 43121 43122 43123 43124 43125 43126 43127 43128 43129 43130 43131 43132 43133 43134 43135 43136 43137 43138 43139 43140 43141 43142 43143 43144 43145 43146 43147 43148 43149 43150 43151 43152 43153 43154 43155 43156 43157 43158 43159 43160 43161 43162 43163 43164 43165 43166 43167 43168 43169 43170 43171 43172 43173 43174 43175 43176 43177 43178 43179 43180 43181 43182 43183 43184 43185 43186 43187 43188 43189 43190 43191 43192 43193 43194 43195 43196 43197 43198 43199 43200 43201 43202 43203 43204 43205 43206 43207 43208 43209 43210 43211 43212 43213 43214 43215 43216 43217 43218 43219 43220 43221 43222 43223 43224 43225 43226 43227 43228 43229 43230 43231 43232 43233 43234 43235 43236 43237 43238 43239 43240 43241 43242 43243 43244 43245 43246 43247 43248 43249 43250 43251 43252 43253 43254 43255 43256 43257 43258 43259 43260 43261 43262 43263 43264 43265 43266 43267 43268 43269 43270 43271 43272 43273 43274 43275 43276 43277 43278 43279 43280 43281 43282 43283 43284 43285 43286 43287 43288 43289 43290 43291 43292 43293 43294 43295 43296 43297 43298 43299 43300 43301 43302 43303 43304 43305 43306 43307 43308 43309 43310 43311 43312 43313 43314 43315 43316 43317 43318 43319 43320 43321 43322 43323 43324 43325 43326 43327 43328 43329 43330 43331 43332 43333 43334 43335 43336 43337 43338 43339 43340 43341 43342 43343 43344 43345 43346 43347 43348 43349 43350 43351 43352 43353 43354 43355 43356 43357 43358 43359 43360 43361 43362 43363 43364 43365 43366 43367 43368 43369 43370 43371 43372 43373 43374 43375 43376 43377 43378 43379 43380 43381 43382 43383 43384 43385 43386 43387 43388 43389 43390 43391 43392 43393 43394 43395 43396 43397 43398 43399 43400 43401 43402 43403 43404 43405 43406 43407 43408 43409 43410 43411 43412 43413 43414 43415 43416 43417 43418 43419 43420 43421 43422 43423 43424 43425 43426 43427 43428 43429 43430 43431 43432 43433 43434 43435 43436 43437 43438 43439 43440 43441 43442 43443 43444 43445 43446 43447 43448 43449 43450 43451 43452 43453 43454 43455 43456 43457 43458 43459 43460 43461 43462 43463 43464 43465 43466 43467 43468 43469 43470 43471 43472 43473 43474 43475 43476 43477 43478 43479 43480 43481 43482 43483 43484 43485 43486 43487 43488 43489 43490 43491 43492 43493 43494 43495 43496 43497 43498 43499 43500 43501 43502 43503 43504 43505 43506 43507 43508 43509 43510 43511 43512 43513 43514 43515 43516 43517 43518 43519 43520 43521 43522 43523 43524 43525 43526 43527 43528 43529 43530 43531 43532 43533 43534 43535 43536 43537 43538 43539 43540 43541 43542 43543 43544 43545 43546 43547 43548 43549 43550 43551 43552 43553 43554 43555 43556 43557 43558 43559 43560 43561 43562 43563 43564 43565 43566 43567 43568 43569 43570 43571 43572 43573 43574 43575 43576 43577 43578 43579 43580 43581 43582 43583 43584 43585 43586 43587 43588 43589 43590 43591 43592 43593 43594 43595 43596 43597 43598 43599 43600 43601 43602 43603 43604 43605 43606 43607 43608 43609 43610 43611 43612 43613 43614 43615 43616 43617 43618 43619 43620 43621 43622 43623 43624 43625 43626 43627 43628 43629 43630 43631 43632 43633 43634 43635 43636 43637 43638 43639 43640 43641 43642 43643 43644 43645 43646 43647 43648 43649 43650 43651 43652 43653 43654 43655 43656 43657 43658 43659 43660 43661 43662 43663 43664 43665 43666 43667 43668 43669 43670 43671 43672 43673 43674 43675 43676 43677 43678 43679 43680 43681 43682 43683 43684 43685 43686 43687 43688 43689 43690 43691 43692 43693 43694 43695 43696 43697 43698 43699 43700 43701 43702 43703 43704 43705 43706 43707 43708 43709 43710 43711 43712 43713 43714 43715 43716 43717 43718 43719 43720 43721 43722 43723 43724 43725 43726 43727 43728 43729 43730 43731 43732 43733 43734 43735 43736 43737 43738 43739 43740 43741 43742 43743 43744 43745 43746 43747 43748 43749 43750 43751 43752 43753 43754 43755 43756 43757 43758 43759 43760 43761 43762 43763 43764 43765 43766 43767 43768 43769 43770 43771 43772 43773 43774 43775 43776 43777 43778 43779 43780 43781 43782 43783 43784 43785 43786 43787 43788 43789 43790 43791 43792 43793 43794 43795 43796 43797 43798 43799 43800 43801 43802 43803 43804 43805 43806 43807 43808 43809 43810 43811 43812 43813 43814 43815 43816 43817 43818 43819 43820 43821 43822 43823 43824 43825 43826 43827 43828 43829 43830 43831 43832 43833 43834 43835 43836 43837 43838 43839 43840 43841 43842 43843 43844 43845 43846 43847 43848 43849 43850 43851 43852 43853 43854 43855 43856 43857 43858 43859 43860 43861 43862 43863 43864 43865 43866 43867 43868 43869 43870 43871 43872 43873 43874 43875 43876 43877 43878 43879 43880 43881 43882 43883 43884 43885 43886 43887 43888 43889 43890 43891 43892 43893 43894 43895 43896 43897 43898 43899 43900 43901 43902 43903 43904 43905 43906 43907 43908 43909 43910 43911 43912 43913 43914 43915 43916 43917 43918 43919 43920 43921 43922 43923 43924 43925 43926 43927 43928 43929 43930 43931 43932 43933 43934 43935 43936 43937 43938 43939 43940 43941 43942 43943 43944 43945 43946 43947 43948 43949 43950 43951 43952 43953 43954 43955 43956 43957 43958 43959 43960 43961 43962 43963 43964 43965 43966 43967 43968 43969 43970 43971 43972 43973 43974 43975 43976 43977 43978 43979 43980 43981 43982 43983 43984 43985 43986 43987 43988 43989 43990 43991 43992 43993 43994 43995 43996 43997 43998 43999 44000 44001 44002 44003 44004 44005 44006 44007 44008 44009 44010 44011 44012 44013 44014 44015 44016 44017 44018 44019 44020 44021 44022 44023 44024 44025 44026 44027 44028 44029 44030 44031 44032 44033 44034 44035 44036 44037 44038 44039 44040 44041 44042 44043 44044 44045 44046 44047 44048 44049 44050 44051 44052 44053 44054 44055 44056 44057 44058 44059 44060 44061 44062 44063 44064 44065 44066 44067 44068 44069 44070 44071 44072 44073 44074 44075 44076 44077 44078 44079 44080 44081 44082 44083 44084 44085 44086 44087 44088 44089 44090 44091 44092 44093 44094 44095 44096 44097 44098 44099 44100 44101 44102 44103 44104 44105 44106 44107 44108 44109 44110 44111 44112 44113 44114 44115 44116 44117 44118 44119 44120 44121 44122 44123 44124 44125 44126 44127 44128 44129 44130 44131 44132 44133 44134 44135 44136 44137 44138 44139 44140 44141 44142 44143 44144 44145 44146 44147 44148 44149 44150 44151 44152 44153 44154 44155 44156 44157 44158 44159 44160 44161 44162 44163 44164 44165 44166 44167 44168 44169 44170 44171 44172 44173 44174 44175 44176 44177 44178 44179 44180 44181 44182 44183 44184 44185 44186 44187 44188 44189 44190 44191 44192 44193 44194 44195 44196 44197 44198 44199 44200 44201 44202 44203 44204 44205 44206 44207 44208 44209 44210 44211 44212 44213 44214 44215 44216 44217 44218 44219 44220 44221 44222 44223 44224 44225 44226 44227 44228 44229 44230 44231 44232 44233 44234 44235 44236 44237 44238 44239 44240 44241 44242 44243 44244 44245 44246 44247 44248 44249 44250 44251 44252 44253 44254 44255 44256 44257 44258 44259 44260 44261 44262 44263 44264 44265 44266 44267 44268 44269 44270 44271 44272 44273 44274 44275 44276 44277 44278 44279 44280 44281 44282 44283 44284 44285 44286 44287 44288 44289 44290 44291 44292 44293 44294 44295 44296 44297 44298 44299 44300 44301 44302 44303 44304 44305 44306 44307 44308 44309 44310 44311 44312 44313 44314 44315 44316 44317 44318 44319 44320 44321 44322 44323 44324 44325 44326 44327 44328 44329 44330 44331 44332 44333 44334 44335 44336 44337 44338 44339 44340 44341 44342 44343 44344 44345 44346 44347 44348 44349 44350 44351 44352 44353 44354 44355 44356 44357 44358 44359 44360 44361 44362 44363 44364 44365 44366 44367 44368 44369 44370 44371 44372 44373 44374 44375 44376 44377 44378 44379 44380 44381 44382 44383 44384 44385 44386 44387 44388 44389 44390 44391 44392 44393 44394 44395 44396 44397 44398 44399 44400 44401 44402 44403 44404 44405 44406 44407 44408 44409 44410 44411 44412 44413 44414 44415 44416 44417 44418 44419 44420 44421 44422 44423 44424 44425 44426 44427 44428 44429 44430 44431 44432 44433 44434 44435 44436 44437 44438 44439 44440 44441 44442 44443 44444 44445 44446 44447 44448 44449 44450 44451 44452 44453 44454 44455 44456 44457 44458 44459 44460 44461 44462 44463 44464 44465 44466 44467 44468 44469 44470 44471 44472 44473 44474 44475 44476 44477 44478 44479 44480 44481 44482 44483 44484 44485 44486 44487 44488 44489 44490 44491 44492 44493 44494 44495 44496 44497 44498 44499 44500 44501 44502 44503 44504 44505 44506 44507 44508 44509 44510 44511 44512 44513 44514 44515 44516 44517 44518 44519 44520 44521 44522 44523 44524 44525 44526 44527 44528 44529 44530 44531 44532 44533 44534 44535 44536 44537 44538 44539 44540 44541 44542 44543 44544 44545 44546 44547 44548 44549 44550 44551 44552 44553 44554 44555 44556 44557 44558 44559 44560 44561 44562 44563 44564 44565 44566 44567 44568 44569 44570 44571 44572 44573 44574 44575 44576 44577 44578 44579 44580 44581 44582 44583 44584 44585 44586 44587 44588 44589 44590 44591 44592 44593 44594 44595 44596 44597 44598 44599 44600 44601 44602 44603 44604 44605 44606 44607 44608 44609 44610 44611 44612 44613 44614 44615 44616 44617 44618 44619 44620 44621 44622 44623 44624 44625 44626 44627 44628 44629 44630 44631 44632 44633 44634 44635 44636 44637 44638 44639 44640 44641 44642 44643 44644 44645 44646 44647 44648 44649 44650 44651 44652 44653 44654 44655 44656 44657 44658 44659 44660 44661 44662 44663 44664 44665 44666 44667 44668 44669 44670 44671 44672 44673 44674 44675 44676 44677 44678 44679 44680 44681 44682 44683 44684 44685 44686 44687 44688 44689 44690 44691 44692 44693 44694 44695 44696 44697 44698 44699 44700 44701 44702 44703 44704 44705 44706 44707 44708 44709 44710 44711 44712 44713 44714 44715 44716 44717 44718 44719 44720 44721 44722 44723 44724 44725 44726 44727 44728 44729 44730 44731 44732 44733 44734 44735 44736 44737 44738 44739 44740 44741 44742 44743 44744 44745 44746 44747 44748 44749 44750 44751 44752 44753 44754 44755 44756 44757 44758 44759 44760 44761 44762 44763 44764 44765 44766 44767 44768 44769 44770 44771 44772 44773 44774 44775 44776 44777 44778 44779 44780 44781 44782 44783 44784 44785 44786 44787 44788 44789 44790 44791 44792 44793 44794 44795 44796 44797 44798 44799 44800 44801 44802 44803 44804 44805 44806 44807 44808 44809 44810 44811 44812 44813 44814 44815 44816 44817 44818 44819 44820 44821 44822 44823 44824 44825 44826 44827 44828 44829 44830 44831 44832 44833 44834 44835 44836 44837 44838 44839 44840 44841 44842 44843 44844 44845 44846 44847 44848 44849 44850 44851 44852 44853 44854 44855 44856 44857 44858 44859 44860 44861 44862 44863 44864 44865 44866 44867 44868 44869 44870 44871 44872 44873 44874 44875 44876 44877 44878 44879 44880 44881 44882 44883 44884 44885 44886 44887 44888 44889 44890 44891 44892 44893 44894 44895 44896 44897 44898 44899 44900 44901 44902 44903 44904 44905 44906 44907 44908 44909 44910 44911 44912 44913 44914 44915 44916 44917 44918 44919 44920 44921 44922 44923 44924 44925 44926 44927 44928 44929 44930 44931 44932 44933 44934 44935 44936 44937 44938 44939 44940 44941 44942 44943 44944 44945 44946 44947 44948 44949 44950 44951 44952 44953 44954 44955 44956 44957 44958 44959 44960 44961 44962 44963 44964 44965 44966 44967 44968 44969 44970 44971 44972 44973 44974 44975 44976 44977 44978 44979 44980 44981 44982 44983 44984 44985 44986 44987 44988 44989 44990 44991 44992 44993 44994 44995 44996 44997 44998 44999 45000 45001 45002 45003 45004 45005 45006 45007 45008 45009 45010 45011 45012 45013 45014 45015 45016 45017 45018 45019 45020 45021 45022 45023 45024 45025 45026 45027 45028 45029 45030 45031 45032 45033 45034 45035 45036 45037 45038 45039 45040 45041 45042 45043 45044 45045 45046 45047 45048 45049 45050 45051 45052 45053 45054 45055 45056 45057 45058 45059 45060 45061 45062 45063 45064 45065 45066 45067 45068 45069 45070 45071 45072 45073 45074 45075 45076 45077 45078 45079 45080 45081 45082 45083 45084 45085 45086 45087 45088 45089 45090 45091 45092 45093 45094 45095 45096 45097 45098 45099 45100 45101 45102 45103 45104 45105 45106 45107 45108 45109 45110 45111 45112 45113 45114 45115 45116 45117 45118 45119 45120 45121 45122 45123 45124 45125 45126 45127 45128 45129 45130 45131 45132 45133 45134 45135 45136 45137 45138 45139 45140 45141 45142 45143 45144 45145 45146 45147 45148 45149 45150 45151 45152 45153 45154 45155 45156 45157 45158 45159 45160 45161 45162 45163 45164 45165 45166 45167 45168 45169 45170 45171 45172 45173 45174 45175 45176 45177 45178 45179 45180 45181 45182 45183 45184 45185 45186 45187 45188 45189 45190 45191 45192 45193 45194 45195 45196 45197 45198 45199 45200 45201 45202 45203 45204 45205 45206 45207 45208 45209 45210 45211 45212 45213 45214 45215 45216 45217 45218 45219 45220 45221 45222 45223 45224 45225 45226 45227 45228 45229 45230 45231 45232 45233 45234 45235 45236 45237 45238 45239 45240 45241 45242 45243 45244 45245 45246 45247 45248 45249 45250 45251 45252 45253 45254 45255 45256 45257 45258 45259 45260 45261 45262 45263 45264 45265 45266 45267 45268 45269 45270 45271 45272 45273 45274 45275 45276 45277 45278 45279 45280 45281 45282 45283 45284 45285 45286 45287 45288 45289 45290 45291 45292 45293 45294 45295 45296 45297 45298 45299 45300 45301 45302 45303 45304 45305 45306 45307 45308 45309 45310 45311 45312 45313 45314 45315 45316 45317 45318 45319 45320 45321 45322 45323 45324 45325 45326 45327 45328 45329 45330 45331 45332 45333 45334 45335 45336 45337 45338 45339 45340 45341 45342 45343 45344 45345 45346 45347 45348 45349 45350 45351 45352 45353 45354 45355 45356 45357 45358 45359 45360 45361 45362 45363 45364 45365 45366 45367 45368 45369 45370 45371 45372 45373 45374 45375 45376 45377 45378 45379 45380 45381 45382 45383 45384 45385 45386 45387 45388 45389 45390 45391 45392 45393 45394 45395 45396 45397 45398 45399 45400 45401 45402 45403 45404 45405 45406 45407 45408 45409 45410 45411 45412 45413 45414 45415 45416 45417 45418 45419 45420 45421 45422 45423 45424 45425 45426 45427 45428 45429 45430 45431 45432 45433 45434 45435 45436 45437 45438 45439 45440 45441 45442 45443 45444 45445 45446 45447 45448 45449 45450 45451 45452 45453 45454 45455 45456 45457 45458 45459 45460 45461 45462 45463 45464 45465 45466 45467 45468 45469 45470 45471 45472 45473 45474 45475 45476 45477 45478 45479 45480 45481 45482 45483 45484 45485 45486 45487 45488 45489 45490 45491 45492 45493 45494 45495 45496 45497 45498 45499 45500 45501 45502 45503 45504 45505 45506 45507 45508 45509 45510 45511 45512 45513 45514 45515 45516 45517 45518 45519 45520 45521 45522 45523 45524 45525 45526 45527 45528 45529 45530 45531 45532 45533 45534 45535 45536 45537 45538 45539 45540 45541 45542 45543 45544 45545 45546 45547 45548 45549 45550 45551 45552 45553 45554 45555 45556 45557 45558 45559 45560 45561 45562 45563 45564 45565 45566 45567 45568 45569 45570 45571 45572 45573 45574 45575 45576 45577 45578 45579 45580 45581 45582 45583 45584 45585 45586 45587 45588 45589 45590 45591 45592 45593 45594 45595 45596 45597 45598 45599 45600 45601 45602 45603 45604 45605 45606 45607 45608 45609 45610 45611 45612 45613 45614 45615 45616 45617 45618 45619 45620 45621 45622 45623 45624 45625 45626 45627 45628 45629 45630 45631 45632 45633 45634 45635 45636 45637 45638 45639 45640 45641 45642 45643 45644 45645 45646 45647 45648 45649 45650 45651 45652 45653 45654 45655 45656 45657 45658 45659 45660 45661 45662 45663 45664 45665 45666 45667 45668 45669 45670 45671 45672 45673 45674 45675 45676 45677 45678 45679 45680 45681 45682 45683 45684 45685 45686 45687 45688 45689 45690 45691 45692 45693 45694 45695 45696 45697 45698 45699 45700 45701 45702 45703 45704 45705 45706 45707 45708 45709 45710 45711 45712 45713 45714 45715 45716 45717 45718 45719 45720 45721 45722 45723 45724 45725 45726 45727 45728 45729 45730 45731 45732 45733 45734 45735 45736 45737 45738 45739 45740 45741 45742 45743 45744 45745 45746 45747 45748 45749 45750 45751 45752 45753 45754 45755 45756 45757 45758 45759 45760 45761 45762 45763 45764 45765 45766 45767 45768 45769 45770 45771 45772 45773 45774 45775 45776 45777 45778 45779 45780 45781 45782 45783 45784 45785 45786 45787 45788 45789 45790 45791 45792 45793 45794 45795 45796 45797 45798 45799 45800 45801 45802 45803 45804 45805 45806 45807 45808 45809 45810 45811 45812 45813 45814 45815 45816 45817 45818 45819 45820 45821 45822 45823 45824 45825 45826 45827 45828 45829 45830 45831 45832 45833 45834 45835 45836 45837 45838 45839 45840 45841 45842 45843 45844 45845 45846 45847 45848 45849 45850 45851 45852 45853 45854 45855 45856 45857 45858 45859 45860 45861 45862 45863 45864 45865 45866 45867 45868 45869 45870 45871 45872 45873 45874 45875 45876 45877 45878 45879 45880 45881 45882 45883 45884 45885 45886 45887 45888 45889 45890 45891 45892 45893 45894 45895 45896 45897 45898 45899 45900 45901 45902 45903 45904 45905 45906 45907 45908 45909 45910 45911 45912 45913 45914 45915 45916 45917 45918 45919 45920 45921 45922 45923 45924 45925 45926 45927 45928 45929 45930 45931 45932 45933 45934 45935 45936 45937 45938 45939 45940 45941 45942 45943 45944 45945 45946 45947 45948 45949 45950 45951 45952 45953 45954 45955 45956 45957 45958 45959 45960 45961 45962 45963 45964 45965 45966 45967 45968 45969 45970 45971 45972 45973 45974 45975 45976 45977 45978 45979 45980 45981 45982 45983 45984 45985 45986 45987 45988 45989 45990 45991 45992 45993 45994 45995 45996 45997 45998 45999 46000 46001 46002 46003 46004 46005 46006 46007 46008 46009 46010 46011 46012 46013 46014 46015 46016 46017 46018 46019 46020 46021 46022 46023 46024 46025 46026 46027 46028 46029 46030 46031 46032 46033 46034 46035 46036 46037 46038 46039 46040 46041 46042 46043 46044 46045 46046 46047 46048 46049 46050 46051 46052 46053 46054 46055 46056 46057 46058 46059 46060 46061 46062 46063 46064 46065 46066 46067 46068 46069 46070 46071 46072 46073 46074 46075 46076 46077 46078 46079 46080 46081 46082 46083 46084 46085 46086 46087 46088 46089 46090 46091 46092 46093 46094 46095 46096 46097 46098 46099 46100 46101 46102 46103 46104 46105 46106 46107 46108 46109 46110 46111 46112 46113 46114 46115 46116 46117 46118 46119 46120 46121 46122 46123 46124 46125 46126 46127 46128 46129 46130 46131 46132 46133 46134 46135 46136 46137 46138 46139 46140 46141 46142 46143 46144 46145 46146 46147 46148 46149 46150 46151 46152 46153 46154 46155 46156 46157 46158 46159 46160 46161 46162 46163 46164 46165 46166 46167 46168 46169 46170 46171 46172 46173 46174 46175 46176 46177 46178 46179 46180 46181 46182 46183 46184 46185 46186 46187 46188 46189 46190 46191 46192 46193 46194 46195 46196 46197 46198 46199 46200 46201 46202 46203 46204 46205 46206 46207 46208 46209 46210 46211 46212 46213 46214 46215 46216 46217 46218 46219 46220 46221 46222 46223 46224 46225 46226 46227 46228 46229 46230 46231 46232 46233 46234 46235 46236 46237 46238 46239 46240 46241 46242 46243 46244 46245 46246 46247 46248 46249 46250 46251 46252 46253 46254 46255 46256 46257 46258 46259 46260 46261 46262 46263 46264 46265 46266 46267 46268 46269 46270 46271 46272 46273 46274 46275 46276 46277 46278 46279 46280 46281 46282 46283 46284 46285 46286 46287 46288 46289 46290 46291 46292 46293 46294 46295 46296 46297 46298 46299 46300 46301 46302 46303 46304 46305 46306 46307 46308 46309 46310 46311 46312 46313 46314 46315 46316 46317 46318 46319 46320 46321 46322 46323 46324 46325 46326 46327 46328 46329 46330 46331 46332 46333 46334 46335 46336 46337 46338 46339 46340 46341 46342 46343 46344 46345 46346 46347 46348 46349 46350 46351 46352 46353 46354 46355 46356 46357 46358 46359 46360 46361 46362 46363 46364 46365 46366 46367 46368 46369 46370 46371 46372 46373 46374 46375 46376 46377 46378 46379 46380 46381 46382 46383 46384 46385 46386 46387 46388 46389 46390 46391 46392 46393 46394 46395 46396 46397 46398 46399 46400 46401 46402 46403 46404 46405 46406 46407 46408 46409 46410 46411 46412 46413 46414 46415 46416 46417 46418 46419 46420 46421 46422 46423 46424 46425 46426 46427 46428 46429 46430 46431 46432 46433 46434 46435 46436 46437 46438 46439 46440 46441 46442 46443 46444 46445 46446 46447 46448 46449 46450 46451 46452 46453 46454 46455 46456 46457 46458 46459 46460 46461 46462 46463 46464 46465 46466 46467 46468 46469 46470 46471 46472 46473 46474 46475 46476 46477 46478 46479 46480 46481 46482 46483 46484 46485 46486 46487 46488 46489 46490 46491 46492 46493 46494 46495 46496 46497 46498 46499 46500 46501 46502 46503 46504 46505 46506 46507 46508 46509 46510 46511 46512 46513 46514 46515 46516 46517 46518 46519 46520 46521 46522 46523 46524 46525 46526 46527 46528 46529 46530 46531 46532 46533 46534 46535 46536 46537 46538 46539 46540 46541 46542 46543 46544 46545 46546 46547 46548 46549 46550 46551 46552 46553 46554 46555 46556 46557 46558 46559 46560 46561 46562 46563 46564 46565 46566 46567 46568 46569 46570 46571 46572 46573 46574 46575 46576 46577 46578 46579 46580 46581 46582 46583 46584 46585 46586 46587 46588 46589 46590 46591 46592 46593 46594 46595 46596 46597 46598 46599 46600 46601 46602 46603 46604 46605 46606 46607 46608 46609 46610 46611 46612 46613 46614 46615 46616 46617 46618 46619 46620 46621 46622 46623 46624 46625 46626 46627 46628 46629 46630 46631 46632 46633 46634 46635 46636 46637 46638 46639 46640 46641 46642 46643 46644 46645 46646 46647 46648 46649 46650 46651 46652 46653 46654 46655 46656 46657 46658 46659 46660 46661 46662 46663 46664 46665 46666 46667 46668 46669 46670 46671 46672 46673 46674 46675 46676 46677 46678 46679 46680 46681 46682 46683 46684 46685 46686 46687 46688 46689 46690 46691 46692 46693 46694 46695 46696 46697 46698 46699 46700 46701 46702 46703 46704 46705 46706 46707 46708 46709 46710 46711 46712 46713 46714 46715 46716 46717 46718 46719 46720 46721 46722 46723 46724 46725 46726 46727 46728 46729 46730 46731 46732 46733 46734 46735 46736 46737 46738 46739 46740 46741 46742 46743 46744 46745 46746 46747 46748 46749 46750 46751 46752 46753 46754 46755 46756 46757 46758 46759 46760 46761 46762 46763 46764 46765 46766 46767 46768 46769 46770 46771 46772 46773 46774 46775 46776 46777 46778 46779 46780 46781 46782 46783 46784 46785 46786 46787 46788 46789 46790 46791 46792 46793 46794 46795 46796 46797 46798 46799 46800 46801 46802 46803 46804 46805 46806 46807 46808 46809 46810 46811 46812 46813 46814 46815 46816 46817 46818 46819 46820 46821 46822 46823 46824 46825 46826 46827 46828 46829 46830 46831 46832 46833 46834 46835 46836 46837 46838 46839 46840 46841 46842 46843 46844 46845 46846 46847 46848 46849 46850 46851 46852 46853 46854 46855 46856 46857 46858 46859 46860 46861 46862 46863 46864 46865 46866 46867 46868 46869 46870 46871 46872 46873 46874 46875 46876 46877 46878 46879 46880 46881 46882 46883 46884 46885 46886 46887 46888 46889 46890 46891 46892 46893 46894 46895 46896 46897 46898 46899 46900 46901 46902 46903 46904 46905 46906 46907 46908 46909 46910 46911 46912 46913 46914 46915 46916 46917 46918 46919 46920 46921 46922 46923 46924 46925 46926 46927 46928 46929 46930 46931 46932 46933 46934 46935 46936 46937 46938 46939 46940 46941 46942 46943 46944 46945 46946 46947 46948 46949 46950 46951 46952 46953 46954 46955 46956 46957 46958 46959 46960 46961 46962 46963 46964 46965 46966 46967 46968 46969 46970 46971 46972 46973 46974 46975 46976 46977 46978 46979 46980 46981 46982 46983 46984 46985 46986 46987 46988 46989 46990 46991 46992 46993 46994 46995 46996 46997 46998 46999 47000 47001 47002 47003 47004 47005 47006 47007 47008 47009 47010 47011 47012 47013 47014 47015 47016 47017 47018 47019 47020 47021 47022 47023 47024 47025 47026 47027 47028 47029 47030 47031 47032 47033 47034 47035 47036 47037 47038 47039 47040 47041 47042 47043 47044 47045 47046 47047 47048 47049 47050 47051 47052 47053 47054 47055 47056 47057 47058 47059 47060 47061 47062 47063 47064 47065 47066 47067 47068 47069 47070 47071 47072 47073 47074 47075 47076 47077 47078 47079 47080 47081 47082 47083 47084 47085 47086 47087 47088 47089 47090 47091 47092 47093 47094 47095 47096 47097 47098 47099 47100 47101 47102 47103 47104 47105 47106 47107 47108 47109 47110 47111 47112 47113 47114 47115 47116 47117 47118 47119 47120 47121 47122 47123 47124 47125 47126 47127 47128 47129 47130 47131 47132 47133 47134 47135 47136 47137 47138 47139 47140 47141 47142 47143 47144 47145 47146 47147 47148 47149 47150 47151 47152 47153 47154 47155 47156 47157 47158 47159 47160 47161 47162 47163 47164 47165 47166 47167 47168 47169 47170 47171 47172 47173 47174 47175 47176 47177 47178 47179 47180 47181 47182 47183 47184 47185 47186 47187 47188 47189 47190 47191 47192 47193 47194 47195 47196 47197 47198 47199 47200 47201 47202 47203 47204 47205 47206 47207 47208 47209 47210 47211 47212 47213 47214 47215 47216 47217 47218 47219 47220 47221 47222 47223 47224 47225 47226 47227 47228 47229 47230 47231 47232 47233 47234 47235 47236 47237 47238 47239 47240 47241 47242 47243 47244 47245 47246 47247 47248 47249 47250 47251 47252 47253 47254 47255 47256 47257 47258 47259 47260 47261 47262 47263 47264 47265 47266 47267 47268 47269 47270 47271 47272 47273 47274 47275 47276 47277 47278 47279 47280 47281 47282 47283 47284 47285 47286 47287 47288 47289 47290 47291 47292 47293 47294 47295 47296 47297 47298 47299 47300 47301 47302 47303 47304 47305 47306 47307 47308 47309 47310 47311 47312 47313 47314 47315 47316 47317 47318 47319 47320 47321 47322 47323 47324 47325 47326 47327 47328 47329 47330 47331 47332 47333 47334 47335 47336 47337 47338 47339 47340 47341 47342 47343 47344 47345 47346 47347 47348 47349 47350 47351 47352 47353 47354 47355 47356 47357 47358 47359 47360 47361 47362 47363 47364 47365 47366 47367 47368 47369 47370 47371 47372 47373 47374 47375 47376 47377 47378 47379 47380 47381 47382 47383 47384 47385 47386 47387 47388 47389 47390 47391 47392 47393 47394 47395 47396 47397 47398 47399 47400 47401 47402 47403 47404 47405 47406 47407 47408 47409 47410 47411 47412 47413 47414 47415 47416 47417 47418 47419 47420 47421 47422 47423 47424 47425 47426 47427 47428 47429 47430 47431 47432 47433 47434 47435 47436 47437 47438 47439 47440 47441 47442 47443 47444 47445 47446 47447 47448 47449 47450 47451 47452 47453 47454 47455 47456 47457 47458 47459 47460 47461 47462 47463 47464 47465 47466 47467 47468 47469 47470 47471 47472 47473 47474 47475 47476 47477 47478 47479 47480 47481 47482 47483 47484 47485 47486 47487 47488 47489 47490 47491 47492 47493 47494 47495 47496 47497 47498 47499 47500 47501 47502 47503 47504 47505 47506 47507 47508 47509 47510 47511 47512 47513 47514 47515 47516 47517 47518 47519 47520 47521 47522 47523 47524 47525 47526 47527 47528 47529 47530 47531 47532 47533 47534 47535 47536 47537 47538 47539 47540 47541 47542 47543 47544 47545 47546 47547 47548 47549 47550 47551 47552 47553 47554 47555 47556 47557 47558 47559 47560 47561 47562 47563 47564 47565 47566 47567 47568 47569 47570 47571 47572 47573 47574 47575 47576 47577 47578 47579 47580 47581 47582 47583 47584 47585 47586 47587 47588 47589 47590 47591 47592 47593 47594 47595 47596 47597 47598 47599 47600 47601 47602 47603 47604 47605 47606 47607 47608 47609 47610 47611 47612 47613 47614 47615 47616 47617 47618 47619 47620 47621 47622 47623 47624 47625 47626 47627 47628 47629 47630 47631 47632 47633 47634 47635 47636 47637 47638 47639 47640 47641 47642 47643 47644 47645 47646 47647 47648 47649 47650 47651 47652 47653 47654 47655 47656 47657 47658 47659 47660 47661 47662 47663 47664 47665 47666 47667 47668 47669 47670 47671 47672 47673 47674 47675 47676 47677 47678 47679 47680 47681 47682 47683 47684 47685 47686 47687 47688 47689 47690 47691 47692 47693 47694 47695 47696 47697 47698 47699 47700 47701 47702 47703 47704 47705 47706 47707 47708 47709 47710 47711 47712 47713 47714 47715 47716 47717 47718 47719 47720 47721 47722 47723 47724 47725 47726 47727 47728 47729 47730 47731 47732 47733 47734 47735 47736 47737 47738 47739 47740 47741 47742 47743 47744 47745 47746 47747 47748 47749 47750 47751 47752 47753 47754 47755 47756 47757 47758 47759 47760 47761 47762 47763 47764 47765 47766 47767 47768 47769 47770 47771 47772 47773 47774 47775 47776 47777 47778 47779 47780 47781 47782 47783 47784 47785 47786 47787 47788 47789 47790 47791 47792 47793 47794 47795 47796 47797 47798 47799 47800 47801 47802 47803 47804 47805 47806 47807 47808 47809 47810 47811 47812 47813 47814 47815 47816 47817 47818 47819 47820 47821 47822 47823 47824 47825 47826 47827 47828 47829 47830 47831 47832 47833 47834 47835 47836 47837 47838 47839 47840 47841 47842 47843 47844 47845 47846 47847 47848 47849 47850 47851 47852 47853 47854 47855 47856 47857 47858 47859 47860 47861 47862 47863 47864 47865 47866 47867 47868 47869 47870 47871 47872 47873 47874 47875 47876 47877 47878 47879 47880 47881 47882 47883 47884 47885 47886 47887 47888 47889 47890 47891 47892 47893 47894 47895 47896 47897 47898 47899 47900 47901 47902 47903 47904 47905 47906 47907 47908 47909 47910 47911 47912 47913 47914 47915 47916 47917 47918 47919 47920 47921 47922 47923 47924 47925 47926 47927 47928 47929 47930 47931 47932 47933 47934 47935 47936 47937 47938 47939 47940 47941 47942 47943 47944 47945 47946 47947 47948 47949 47950 47951 47952 47953 47954 47955 47956 47957 47958 47959 47960 47961 47962 47963 47964 47965 47966 47967 47968 47969 47970 47971 47972 47973 47974 47975 47976 47977 47978 47979 47980 47981 47982 47983 47984 47985 47986 47987 47988 47989 47990 47991 47992 47993 47994 47995 47996 47997 47998 47999 48000 48001 48002 48003 48004 48005 48006 48007 48008 48009 48010 48011 48012 48013 48014 48015 48016 48017 48018 48019 48020 48021 48022 48023 48024 48025 48026 48027 48028 48029 48030 48031 48032 48033 48034 48035 48036 48037 48038 48039 48040 48041 48042 48043 48044 48045 48046 48047 48048 48049 48050 48051 48052 48053 48054 48055 48056 48057 48058 48059 48060 48061 48062 48063 48064 48065 48066 48067 48068 48069 48070 48071 48072 48073 48074 48075 48076 48077 48078 48079 48080 48081 48082 48083 48084 48085 48086 48087 48088 48089 48090 48091 48092 48093 48094 48095 48096 48097 48098 48099 48100 48101 48102 48103 48104 48105 48106 48107 48108 48109 48110 48111 48112 48113 48114 48115 48116 48117 48118 48119 48120 48121 48122 48123 48124 48125 48126 48127 48128 48129 48130 48131 48132 48133 48134 48135 48136 48137 48138 48139 48140 48141 48142 48143 48144 48145 48146 48147 48148 48149 48150 48151 48152 48153 48154 48155 48156 48157 48158 48159 48160 48161 48162 48163 48164 48165 48166 48167 48168 48169 48170 48171 48172 48173 48174 48175 48176 48177 48178 48179 48180 48181 48182 48183 48184 48185 48186 48187 48188 48189 48190 48191 48192 48193 48194 48195 48196 48197 48198 48199 48200 48201 48202 48203 48204 48205 48206 48207 48208 48209 48210 48211 48212 48213 48214 48215 48216 48217 48218 48219 48220 48221 48222 48223 48224 48225 48226 48227 48228 48229 48230 48231 48232 48233 48234 48235 48236 48237 48238 48239 48240 48241 48242 48243 48244 48245 48246 48247 48248 48249 48250 48251 48252 48253 48254 48255 48256 48257 48258 48259 48260 48261 48262 48263 48264 48265 48266 48267 48268 48269 48270 48271 48272 48273 48274 48275 48276 48277 48278 48279 48280 48281 48282 48283 48284 48285 48286 48287 48288 48289 48290 48291 48292 48293 48294 48295 48296 48297 48298 48299 48300 48301 48302 48303 48304 48305 48306 48307 48308 48309 48310 48311 48312 48313 48314 48315 48316 48317 48318 48319 48320 48321 48322 48323 48324 48325 48326 48327 48328 48329 48330 48331 48332 48333 48334 48335 48336 48337 48338 48339 48340 48341 48342 48343 48344 48345 48346 48347 48348 48349 48350 48351 48352 48353 48354 48355 48356 48357 48358 48359 48360 48361 48362 48363 48364 48365 48366 48367 48368 48369 48370 48371 48372 48373 48374 48375 48376 48377 48378 48379 48380 48381 48382 48383 48384 48385 48386 48387 48388 48389 48390 48391 48392 48393 48394 48395 48396 48397 48398 48399 48400 48401 48402 48403 48404 48405 48406 48407 48408 48409 48410 48411 48412 48413 48414 48415 48416 48417 48418 48419 48420 48421 48422 48423 48424 48425 48426 48427 48428 48429 48430 48431 48432 48433 48434 48435 48436 48437 48438 48439 48440 48441 48442 48443 48444 48445 48446 48447 48448 48449 48450 48451 48452 48453 48454 48455 48456 48457 48458 48459 48460 48461 48462 48463 48464 48465 48466 48467 48468 48469 48470 48471 48472 48473 48474 48475 48476 48477 48478 48479 48480 48481 48482 48483 48484 48485 48486 48487 48488 48489 48490 48491 48492 48493 48494 48495 48496 48497 48498 48499 48500 48501 48502 48503 48504 48505 48506 48507 48508 48509 48510 48511 48512 48513 48514 48515 48516 48517 48518 48519 48520 48521 48522 48523 48524 48525 48526 48527 48528 48529 48530 48531 48532 48533 48534 48535 48536 48537 48538 48539 48540 48541 48542 48543 48544 48545 48546 48547 48548 48549 48550 48551 48552 48553 48554 48555 48556 48557 48558 48559 48560 48561 48562 48563 48564 48565 48566 48567 48568 48569 48570 48571 48572 48573 48574 48575 48576 48577 48578 48579 48580 48581 48582 48583 48584 48585 48586 48587 48588 48589 48590 48591 48592 48593 48594 48595 48596 48597 48598 48599 48600 48601 48602 48603 48604 48605 48606 48607 48608 48609 48610 48611 48612 48613 48614 48615 48616 48617 48618 48619 48620 48621 48622 48623 48624 48625 48626 48627 48628 48629 48630 48631 48632 48633 48634 48635 48636 48637 48638 48639 48640 48641 48642 48643 48644 48645 48646 48647 48648 48649 48650 48651 48652 48653 48654 48655 48656 48657 48658 48659 48660 48661 48662 48663 48664 48665 48666 48667 48668 48669 48670 48671 48672 48673 48674 48675 48676 48677 48678 48679 48680 48681 48682 48683 48684 48685 48686 48687 48688 48689 48690 48691 48692 48693 48694 48695 48696 48697 48698 48699 48700 48701 48702 48703 48704 48705 48706 48707 48708 48709 48710 48711 48712 48713 48714 48715 48716 48717 48718 48719 48720 48721 48722 48723 48724 48725 48726 48727 48728 48729 48730 48731 48732 48733 48734 48735 48736 48737 48738 48739 48740 48741 48742 48743 48744 48745 48746 48747 48748 48749 48750 48751 48752 48753 48754 48755 48756 48757 48758 48759 48760 48761 48762 48763 48764 48765 48766 48767 48768 48769 48770 48771 48772 48773 48774 48775 48776 48777 48778 48779 48780 48781 48782 48783 48784 48785 48786 48787 48788 48789 48790 48791 48792 48793 48794 48795 48796 48797 48798 48799 48800 48801 48802 48803 48804 48805 48806 48807 48808 48809 48810 48811 48812 48813 48814 48815 48816 48817 48818 48819 48820 48821 48822 48823 48824 48825 48826 48827 48828 48829 48830 48831 48832 48833 48834 48835 48836 48837 48838 48839 48840 48841 48842 48843 48844 48845 48846 48847 48848 48849 48850 48851 48852 48853 48854 48855 48856 48857 48858 48859 48860 48861 48862 48863 48864 48865 48866 48867 48868 48869 48870 48871 48872 48873 48874 48875 48876 48877 48878 48879 48880 48881 48882 48883 48884 48885 48886 48887 48888 48889 48890 48891 48892 48893 48894 48895 48896 48897 48898 48899 48900 48901 48902 48903 48904 48905 48906 48907 48908 48909 48910 48911 48912 48913 48914 48915 48916 48917 48918 48919 48920 48921 48922 48923 48924 48925 48926 48927 48928 48929 48930 48931 48932 48933 48934 48935 48936 48937 48938 48939 48940 48941 48942 48943 48944 48945 48946 48947 48948 48949 48950 48951 48952 48953 48954 48955 48956 48957 48958 48959 48960 48961 48962 48963 48964 48965 48966 48967 48968 48969 48970 48971 48972 48973 48974 48975 48976 48977 48978 48979 48980 48981 48982 48983 48984 48985 48986 48987 48988 48989 48990 48991 48992 48993 48994 48995 48996 48997 48998 48999 49000 49001 49002 49003 49004 49005 49006 49007 49008 49009 49010 49011 49012 49013 49014 49015 49016 49017 49018 49019 49020 49021 49022 49023 49024 49025 49026 49027 49028 49029 49030 49031 49032 49033 49034 49035 49036 49037 49038 49039 49040 49041 49042 49043 49044 49045 49046 49047 49048 49049 49050 49051 49052 49053 49054 49055 49056 49057 49058 49059 49060 49061 49062 49063 49064 49065 49066 49067 49068 49069 49070 49071 49072 49073 49074 49075 49076 49077 49078 49079 49080 49081 49082 49083 49084 49085 49086 49087 49088 49089 49090 49091 49092 49093 49094 49095 49096 49097 49098 49099 49100 49101 49102 49103 49104 49105 49106 49107 49108 49109 49110 49111 49112 49113 49114 49115 49116 49117 49118 49119 49120 49121 49122 49123 49124 49125 49126 49127 49128 49129 49130 49131 49132 49133 49134 49135 49136 49137 49138 49139 49140 49141 49142 49143 49144 49145 49146 49147 49148 49149 49150 49151 49152 49153 49154 49155 49156 49157 49158 49159 49160 49161 49162 49163 49164 49165 49166 49167 49168 49169 49170 49171 49172 49173 49174 49175 49176 49177 49178 49179 49180 49181 49182 49183 49184 49185 49186 49187 49188 49189 49190 49191 49192 49193 49194 49195 49196 49197 49198 49199 49200 49201 49202 49203 49204 49205 49206 49207 49208 49209 49210 49211 49212 49213 49214 49215 49216 49217 49218 49219 49220 49221 49222 49223 49224 49225 49226 49227 49228 49229 49230 49231 49232 49233 49234 49235 49236 49237 49238 49239 49240 49241 49242 49243 49244 49245 49246 49247 49248 49249 49250 49251 49252 49253 49254 49255 49256 49257 49258 49259 49260 49261 49262 49263 49264 49265 49266 49267 49268 49269 49270 49271 49272 49273 49274 49275 49276 49277 49278 49279 49280 49281 49282 49283 49284 49285 49286 49287 49288 49289 49290 49291 49292 49293 49294 49295 49296 49297 49298 49299 49300 49301 49302 49303 49304 49305 49306 49307 49308 49309 49310 49311 49312 49313 49314 49315 49316 49317 49318 49319 49320 49321 49322 49323 49324 49325 49326 49327 49328 49329 49330 49331 49332 49333 49334 49335 49336 49337 49338 49339 49340 49341 49342 49343 49344 49345 49346 49347 49348 49349 49350 49351 49352 49353 49354 49355 49356 49357 49358 49359 49360 49361 49362 49363 49364 49365 49366 49367 49368 49369 49370 49371 49372 49373 49374 49375 49376 49377 49378 49379 49380 49381 49382 49383 49384 49385 49386 49387 49388 49389 49390 49391 49392 49393 49394 49395 49396 49397 49398 49399 49400 49401 49402 49403 49404 49405 49406 49407 49408 49409 49410 49411 49412 49413 49414 49415 49416 49417 49418 49419 49420 49421 49422 49423 49424 49425 49426 49427 49428 49429 49430 49431 49432 49433 49434 49435 49436 49437 49438 49439 49440 49441 49442 49443 49444 49445 49446 49447 49448 49449 49450 49451 49452 49453 49454 49455 49456 49457 49458 49459 49460 49461 49462 49463 49464 49465 49466 49467 49468 49469 49470 49471 49472 49473 49474 49475 49476 49477 49478 49479 49480 49481 49482 49483 49484 49485 49486 49487 49488 49489 49490 49491 49492 49493 49494 49495 49496 49497 49498 49499 49500 49501 49502 49503 49504 49505 49506 49507 49508 49509 49510 49511 49512 49513 49514 49515 49516 49517 49518 49519 49520 49521 49522 49523 49524 49525 49526 49527 49528 49529 49530 49531 49532 49533 49534 49535 49536 49537 49538 49539 49540 49541 49542 49543 49544 49545 49546 49547 49548 49549 49550 49551 49552 49553 49554 49555 49556 49557 49558 49559 49560 49561 49562 49563 49564 49565 49566 49567 49568 49569 49570 49571 49572 49573 49574 49575 49576 49577 49578 49579 49580 49581 49582 49583 49584 49585 49586 49587 49588 49589 49590 49591 49592 49593 49594 49595 49596 49597 49598 49599 49600 49601 49602 49603 49604 49605 49606 49607 49608 49609 49610 49611 49612 49613 49614 49615 49616 49617 49618 49619 49620 49621 49622 49623 49624 49625 49626 49627 49628 49629 49630 49631 49632 49633 49634 49635 49636 49637 49638 49639 49640 49641 49642 49643 49644 49645 49646 49647 49648 49649 49650 49651 49652 49653 49654 49655 49656 49657 49658 49659 49660 49661 49662 49663 49664 49665 49666 49667 49668 49669 49670 49671 49672 49673 49674 49675 49676 49677 49678 49679 49680 49681 49682 49683 49684 49685 49686 49687 49688 49689 49690 49691 49692 49693 49694 49695 49696 49697 49698 49699 49700 49701 49702 49703 49704 49705 49706 49707 49708 49709 49710 49711 49712 49713 49714 49715 49716 49717 49718 49719 49720 49721 49722 49723 49724 49725 49726 49727 49728 49729 49730 49731 49732 49733 49734 49735 49736 49737 49738 49739 49740 49741 49742 49743 49744 49745 49746 49747 49748 49749 49750 49751 49752 49753 49754 49755 49756 49757 49758 49759 49760 49761 49762 49763 49764 49765 49766 49767 49768 49769 49770 49771 49772 49773 49774 49775 49776 49777 49778 49779 49780 49781 49782 49783 49784 49785 49786 49787 49788 49789 49790 49791 49792 49793 49794 49795 49796 49797 49798 49799 49800 49801 49802 49803 49804 49805 49806 49807 49808 49809 49810 49811 49812 49813 49814 49815 49816 49817 49818 49819 49820 49821 49822 49823 49824 49825 49826 49827 49828 49829 49830 49831 49832 49833 49834 49835 49836 49837 49838 49839 49840 49841 49842 49843 49844 49845 49846 49847 49848 49849 49850 49851 49852 49853 49854 49855 49856 49857 49858 49859 49860 49861 49862 49863 49864 49865 49866 49867 49868 49869 49870 49871 49872 49873 49874 49875 49876 49877 49878 49879 49880 49881 49882 49883 49884 49885 49886 49887 49888 49889 49890 49891 49892 49893 49894 49895 49896 49897 49898 49899 49900 49901 49902 49903 49904 49905 49906 49907 49908 49909 49910 49911 49912 49913 49914 49915 49916 49917 49918 49919 49920 49921 49922 49923 49924 49925 49926 49927 49928 49929 49930 49931 49932 49933 49934 49935 49936 49937 49938 49939 49940 49941 49942 49943 49944 49945 49946 49947 49948 49949 49950 49951 49952 49953 49954 49955 49956 49957 49958 49959 49960 49961 49962 49963 49964 49965 49966 49967 49968 49969 49970 49971 49972 49973 49974 49975 49976 49977 49978 49979 49980 49981 49982 49983 49984 49985 49986 49987 49988 49989 49990 49991 49992 49993 49994 49995 49996 49997 49998 49999 50000 50001 50002 50003 50004 50005 50006 50007 50008 50009 50010 50011 50012 50013 50014 50015 50016 50017 50018 50019 50020 50021 50022 50023 50024 50025 50026 50027 50028 50029 50030 50031 50032 50033 50034 50035 50036 50037 50038 50039 50040 50041 50042 50043 50044 50045 50046 50047 50048 50049 50050 50051 50052 50053 50054 50055 50056 50057 50058 50059 50060 50061 50062 50063 50064 50065 50066 50067 50068 50069 50070 50071 50072 50073 50074 50075 50076 50077 50078 50079 50080 50081 50082 50083 50084 50085 50086 50087 50088 50089 50090 50091 50092 50093 50094 50095 50096 50097 50098 50099 50100 50101 50102 50103 50104 50105 50106 50107 50108 50109 50110 50111 50112 50113 50114 50115 50116 50117 50118 50119 50120 50121 50122 50123 50124 50125 50126 50127 50128 50129 50130 50131 50132 50133 50134 50135 50136 50137 50138 50139 50140 50141 50142 50143 50144 50145 50146 50147 50148 50149 50150 50151 50152 50153 50154 50155 50156 50157 50158 50159 50160 50161 50162 50163 50164 50165 50166 50167 50168 50169 50170 50171 50172 50173 50174 50175 50176 50177 50178 50179 50180 50181 50182 50183 50184 50185 50186 50187 50188 50189 50190 50191 50192 50193 50194 50195 50196 50197 50198 50199 50200 50201 50202 50203 50204 50205 50206 50207 50208 50209 50210 50211 50212 50213 50214 50215 50216 50217 50218 50219 50220 50221 50222 50223 50224 50225 50226 50227 50228 50229 50230 50231 50232 50233 50234 50235 50236 50237 50238 50239 50240 50241 50242 50243 50244 50245 50246 50247 50248 50249 50250 50251 50252 50253 50254 50255 50256 50257 50258 50259 50260 50261 50262 50263 50264 50265 50266 50267 50268 50269 50270 50271 50272 50273 50274 50275 50276 50277 50278 50279 50280 50281 50282 50283 50284 50285 50286 50287 50288 50289 50290 50291 50292 50293 50294 50295 50296 50297 50298 50299 50300 50301 50302 50303 50304 50305 50306 50307 50308 50309 50310 50311 50312 50313 50314 50315 50316 50317 50318 50319 50320 50321 50322 50323 50324 50325 50326 50327 50328 50329 50330 50331 50332 50333 50334 50335 50336 50337 50338 50339 50340 50341 50342 50343 50344 50345 50346 50347 50348 50349 50350 50351 50352 50353 50354 50355 50356 50357 50358 50359 50360 50361 50362 50363 50364 50365 50366 50367 50368 50369 50370 50371 50372 50373 50374 50375 50376 50377 50378 50379 50380 50381 50382 50383 50384 50385 50386 50387 50388 50389 50390 50391 50392 50393 50394 50395 50396 50397 50398 50399 50400 50401 50402 50403 50404 50405 50406 50407 50408 50409 50410 50411 50412 50413 50414 50415 50416 50417 50418 50419 50420 50421 50422 50423 50424 50425 50426 50427 50428 50429 50430 50431 50432 50433 50434 50435 50436 50437 50438 50439 50440 50441 50442 50443 50444 50445 50446 50447 50448 50449 50450 50451 50452 50453 50454 50455 50456 50457 50458 50459 50460 50461 50462 50463 50464 50465 50466 50467 50468 50469 50470 50471 50472 50473 50474 50475 50476 50477 50478 50479 50480 50481 50482 50483 50484 50485 50486 50487 50488 50489 50490 50491 50492 50493 50494 50495 50496 50497 50498 50499 50500 50501 50502 50503 50504 50505 50506 50507 50508 50509 50510 50511 50512 50513 50514 50515 50516 50517 50518 50519 50520 50521 50522 50523 50524 50525 50526 50527 50528 50529 50530 50531 50532 50533 50534 50535 50536 50537 50538 50539 50540 50541 50542 50543 50544 50545 50546 50547 50548 50549 50550 50551 50552 50553 50554 50555 50556 50557 50558 50559 50560 50561 50562 50563 50564 50565 50566 50567 50568 50569 50570 50571 50572 50573 50574 50575 50576 50577 50578 50579 50580 50581 50582 50583 50584 50585 50586 50587 50588 50589 50590 50591 50592 50593 50594 50595 50596 50597 50598 50599 50600 50601 50602 50603 50604 50605 50606 50607 50608 50609 50610 50611 50612 50613 50614 50615 50616 50617 50618 50619 50620 50621 50622 50623 50624 50625 50626 50627 50628 50629 50630 50631 50632 50633 50634 50635 50636 50637 50638 50639 50640 50641 50642 50643 50644 50645 50646 50647 50648 50649 50650 50651 50652 50653 50654 50655 50656 50657 50658 50659 50660 50661 50662 50663 50664 50665 50666 50667 50668 50669 50670 50671 50672 50673 50674 50675 50676 50677 50678 50679 50680 50681 50682 50683 50684 50685 50686 50687 50688 50689 50690 50691 50692 50693 50694 50695 50696 50697 50698 50699 50700 50701 50702 50703 50704 50705 50706 50707 50708 50709 50710 50711 50712 50713 50714 50715 50716 50717 50718 50719 50720 50721 50722 50723 50724 50725 50726 50727 50728 50729 50730 50731 50732 50733 50734 50735 50736 50737 50738 50739 50740 50741 50742 50743 50744 50745 50746 50747 50748 50749 50750 50751 50752 50753 50754 50755 50756 50757 50758 50759 50760 50761 50762 50763 50764 50765 50766 50767 50768 50769 50770 50771 50772 50773 50774 50775 50776 50777 50778 50779 50780 50781 50782 50783 50784 50785 50786 50787 50788 50789 50790 50791 50792 50793 50794 50795 50796 50797 50798 50799 50800 50801 50802 50803 50804 50805 50806 50807 50808 50809 50810 50811 50812 50813 50814 50815 50816 50817 50818 50819 50820 50821 50822 50823 50824 50825 50826 50827 50828 50829 50830 50831 50832 50833 50834 50835 50836 50837 50838 50839 50840 50841 50842 50843 50844 50845 50846 50847 50848 50849 50850 50851 50852 50853 50854 50855 50856 50857 50858 50859 50860 50861 50862 50863 50864 50865 50866 50867 50868 50869 50870 50871 50872 50873 50874 50875 50876 50877 50878 50879 50880 50881 50882 50883 50884 50885 50886 50887 50888 50889 50890 50891 50892 50893 50894 50895 50896 50897 50898 50899 50900 50901 50902 50903 50904 50905 50906 50907 50908 50909 50910 50911 50912 50913 50914 50915 50916 50917 50918 50919 50920 50921 50922 50923 50924 50925 50926 50927 50928 50929 50930 50931 50932 50933 50934 50935 50936 50937 50938 50939 50940 50941 50942 50943 50944 50945 50946 50947 50948 50949 50950 50951 50952 50953 50954 50955 50956 50957 50958 50959 50960 50961 50962 50963 50964 50965 50966 50967 50968 50969 50970 50971 50972 50973 50974 50975 50976 50977 50978 50979 50980 50981 50982 50983 50984 50985 50986 50987 50988 50989 50990 50991 50992 50993 50994 50995 50996 50997 50998 50999 51000 51001 51002 51003 51004 51005 51006 51007 51008 51009 51010 51011 51012 51013 51014 51015 51016 51017 51018 51019 51020 51021 51022 51023 51024 51025 51026 51027 51028 51029 51030 51031 51032 51033 51034 51035 51036 51037 51038 51039 51040 51041 51042 51043 51044 51045 51046 51047 51048 51049 51050 51051 51052 51053 51054 51055 51056 51057 51058 51059 51060 51061 51062 51063 51064 51065 51066 51067 51068 51069 51070 51071 51072 51073 51074 51075 51076 51077 51078 51079 51080 51081 51082 51083 51084 51085 51086 51087 51088 51089 51090 51091 51092 51093 51094 51095 51096 51097 51098 51099 51100 51101 51102 51103 51104 51105 51106 51107 51108 51109 51110 51111 51112 51113 51114 51115 51116 51117 51118 51119 51120 51121 51122 51123 51124 51125 51126 51127 51128 51129 51130 51131 51132 51133 51134 51135 51136 51137 51138 51139 51140 51141 51142 51143 51144 51145 51146 51147 51148 51149 51150 51151 51152 51153 51154 51155 51156 51157 51158 51159 51160 51161 51162 51163 51164 51165 51166 51167 51168 51169 51170 51171 51172 51173 51174 51175 51176 51177 51178 51179 51180 51181 51182 51183 51184 51185 51186 51187 51188 51189 51190 51191 51192 51193 51194 51195 51196 51197 51198 51199 51200 51201 51202 51203 51204 51205 51206 51207 51208 51209 51210 51211 51212 51213 51214 51215 51216 51217 51218 51219 51220 51221 51222 51223 51224 51225 51226 51227 51228 51229 51230 51231 51232 51233 51234 51235 51236 51237 51238 51239 51240 51241 51242 51243 51244 51245 51246 51247 51248 51249 51250 51251 51252 51253 51254 51255 51256 51257 51258 51259 51260 51261 51262 51263 51264 51265 51266 51267 51268 51269 51270 51271 51272 51273 51274 51275 51276 51277 51278 51279 51280 51281 51282 51283 51284 51285 51286 51287 51288 51289 51290 51291 51292 51293 51294 51295 51296 51297 51298 51299 51300 51301 51302 51303 51304 51305 51306 51307 51308 51309 51310 51311 51312 51313 51314 51315 51316 51317 51318 51319 51320 51321 51322 51323 51324 51325 51326 51327 51328 51329 51330 51331 51332 51333 51334 51335 51336 51337 51338 51339 51340 51341 51342 51343 51344 51345 51346 51347 51348 51349 51350 51351 51352 51353 51354 51355 51356 51357 51358 51359 51360 51361 51362 51363 51364 51365 51366 51367 51368 51369 51370 51371 51372 51373 51374 51375 51376 51377 51378 51379 51380 51381 51382 51383 51384 51385 51386 51387 51388 51389 51390 51391 51392 51393 51394 51395 51396 51397 51398 51399 51400 51401 51402 51403 51404 51405 51406 51407 51408 51409 51410 51411 51412 51413 51414 51415 51416 51417 51418 51419 51420 51421 51422 51423 51424 51425 51426 51427 51428 51429 51430 51431 51432 51433 51434 51435 51436 51437 51438 51439 51440 51441 51442 51443 51444 51445 51446 51447 51448 51449 51450 51451 51452 51453 51454 51455 51456 51457 51458 51459 51460 51461 51462 51463 51464 51465 51466 51467 51468 51469 51470 51471 51472 51473 51474 51475 51476 51477 51478 51479 51480 51481 51482 51483 51484 51485 51486 51487 51488 51489 51490 51491 51492 51493 51494 51495 51496 51497 51498 51499 51500 51501 51502 51503 51504 51505 51506 51507 51508 51509 51510 51511 51512 51513 51514 51515 51516 51517 51518 51519 51520 51521 51522 51523 51524 51525 51526 51527 51528 51529 51530 51531 51532 51533 51534 51535 51536 51537 51538 51539 51540 51541 51542 51543 51544 51545 51546 51547 51548 51549 51550 51551 51552 51553 51554 51555 51556 51557 51558 51559 51560 51561 51562 51563 51564 51565 51566 51567 51568 51569 51570 51571 51572 51573 51574 51575 51576 51577 51578 51579 51580 51581 51582 51583 51584 51585 51586 51587 51588 51589 51590 51591 51592 51593 51594 51595 51596 51597 51598 51599 51600 51601 51602 51603 51604 51605 51606 51607 51608 51609 51610 51611 51612 51613 51614 51615 51616 51617 51618 51619 51620 51621 51622 51623 51624 51625 51626 51627 51628 51629 51630 51631 51632 51633 51634 51635 51636 51637 51638 51639 51640 51641 51642 51643 51644 51645 51646 51647 51648 51649 51650 51651 51652 51653 51654 51655 51656 51657 51658 51659 51660 51661 51662 51663 51664 51665 51666 51667 51668 51669 51670 51671 51672 51673 51674 51675 51676 51677 51678 51679 51680 51681 51682 51683 51684 51685 51686 51687 51688 51689 51690 51691 51692 51693 51694 51695 51696 51697 51698 51699 51700 51701 51702 51703 51704 51705 51706 51707 51708 51709 51710 51711 51712 51713 51714 51715 51716 51717 51718 51719 51720 51721 51722 51723 51724 51725 51726 51727 51728 51729 51730 51731 51732 51733 51734 51735 51736 51737 51738 51739 51740 51741 51742 51743 51744 51745 51746 51747 51748 51749 51750 51751 51752 51753 51754 51755 51756 51757 51758 51759 51760 51761 51762 51763 51764 51765 51766 51767 51768 51769 51770 51771 51772 51773 51774 51775 51776 51777 51778 51779 51780 51781 51782 51783 51784 51785 51786 51787 51788 51789 51790 51791 51792 51793 51794 51795 51796 51797 51798 51799 51800 51801 51802 51803 51804 51805 51806 51807 51808 51809 51810 51811 51812 51813 51814 51815 51816 51817 51818 51819 51820 51821 51822 51823 51824 51825 51826 51827 51828 51829 51830 51831 51832 51833 51834 51835 51836 51837 51838 51839 51840 51841 51842 51843 51844 51845 51846 51847 51848 51849 51850 51851 51852 51853 51854 51855 51856 51857 51858 51859 51860 51861 51862 51863 51864 51865 51866 51867 51868 51869 51870 51871 51872 51873 51874 51875 51876 51877 51878 51879 51880 51881 51882 51883 51884 51885 51886 51887 51888 51889 51890 51891 51892 51893 51894 51895 51896 51897 51898 51899 51900 51901 51902 51903 51904 51905 51906 51907 51908 51909 51910 51911 51912 51913 51914 51915 51916 51917 51918 51919 51920 51921 51922 51923 51924 51925 51926 51927 51928 51929 51930 51931 51932 51933 51934 51935 51936 51937 51938 51939 51940 51941 51942 51943 51944 51945 51946 51947 51948 51949 51950 51951 51952 51953 51954 51955 51956 51957 51958 51959 51960 51961 51962 51963 51964 51965 51966 51967 51968 51969 51970 51971 51972 51973 51974 51975 51976 51977 51978 51979 51980 51981 51982 51983 51984 51985 51986 51987 51988 51989 51990 51991 51992 51993 51994 51995 51996 51997 51998 51999 52000 52001 52002 52003 52004 52005 52006 52007 52008 52009 52010 52011 52012 52013 52014 52015 52016 52017 52018 52019 52020 52021 52022 52023 52024 52025 52026 52027 52028 52029 52030 52031 52032 52033 52034 52035 52036 52037 52038 52039 52040 52041 52042 52043 52044 52045 52046 52047 52048 52049 52050 52051 52052 52053 52054 52055 52056 52057 52058 52059 52060 52061 52062 52063 52064 52065 52066 52067 52068 52069 52070 52071 52072 52073 52074 52075 52076 52077 52078 52079 52080 52081 52082 52083 52084 52085 52086 52087 52088 52089 52090 52091 52092 52093 52094 52095 52096 52097 52098 52099 52100 52101 52102 52103 52104 52105 52106 52107 52108 52109 52110 52111 52112 52113 52114 52115 52116 52117 52118 52119 52120 52121 52122 52123 52124 52125 52126 52127 52128 52129 52130 52131 52132 52133 52134 52135 52136 52137 52138 52139 52140 52141 52142 52143 52144 52145 52146 52147 52148 52149 52150 52151 52152 52153 52154 52155 52156 52157 52158 52159 52160 52161 52162 52163 52164 52165 52166 52167 52168 52169 52170 52171 52172 52173 52174 52175 52176 52177 52178 52179 52180 52181 52182 52183 52184 
/* Subroutines used for code generation on IA-32.
   Copyright (C) 1988-2017 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.

GCC is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "rtl.h"
#include "tree.h"
#include "memmodel.h"
#include "gimple.h"
#include "cfghooks.h"
#include "cfgloop.h"
#include "df.h"
#include "tm_p.h"
#include "stringpool.h"
#include "expmed.h"
#include "optabs.h"
#include "regs.h"
#include "emit-rtl.h"
#include "recog.h"
#include "cgraph.h"
#include "diagnostic.h"
#include "cfgbuild.h"
#include "alias.h"
#include "fold-const.h"
#include "attribs.h"
#include "calls.h"
#include "stor-layout.h"
#include "varasm.h"
#include "output.h"
#include "insn-attr.h"
#include "flags.h"
#include "except.h"
#include "explow.h"
#include "expr.h"
#include "cfgrtl.h"
#include "common/common-target.h"
#include "langhooks.h"
#include "reload.h"
#include "gimplify.h"
#include "dwarf2.h"
#include "tm-constrs.h"
#include "params.h"
#include "cselib.h"
#include "sched-int.h"
#include "opts.h"
#include "tree-pass.h"
#include "context.h"
#include "pass_manager.h"
#include "target-globals.h"
#include "gimple-iterator.h"
#include "tree-vectorizer.h"
#include "shrink-wrap.h"
#include "builtins.h"
#include "rtl-iter.h"
#include "tree-iterator.h"
#include "tree-chkp.h"
#include "rtl-chkp.h"
#include "dbgcnt.h"
#include "case-cfn-macros.h"
#include "regrename.h"
#include "dojump.h"
#include "fold-const-call.h"
#include "tree-vrp.h"
#include "tree-ssanames.h"
#include "selftest.h"
#include "selftest-rtl.h"
#include "print-rtl.h"
#include "intl.h"

/* This file should be included last.  */
#include "target-def.h"

static rtx legitimize_dllimport_symbol (rtx, bool);
static rtx legitimize_pe_coff_extern_decl (rtx, bool);
static rtx legitimize_pe_coff_symbol (rtx, bool);
static void ix86_print_operand_address_as (FILE *, rtx, addr_space_t, bool);

#ifndef CHECK_STACK_LIMIT
#define CHECK_STACK_LIMIT (-1)
#endif

/* Return index of given mode in mult and division cost tables.  */
#define MODE_INDEX(mode)					\
  ((mode) == QImode ? 0						\
   : (mode) == HImode ? 1					\
   : (mode) == SImode ? 2					\
   : (mode) == DImode ? 3					\
   : 4)

/* Processor costs (relative to an add) */
/* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes.  */
#define COSTS_N_BYTES(N) ((N) * 2)

#define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall, false}}}

static stringop_algs ix86_size_memcpy[2] = {
  {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
  {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}}};
static stringop_algs ix86_size_memset[2] = {
  {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
  {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}}};

const
struct processor_costs ix86_size_cost = {/* costs for tuning for size */
  COSTS_N_BYTES (2),			/* cost of an add instruction */
  COSTS_N_BYTES (3),			/* cost of a lea instruction */
  COSTS_N_BYTES (2),			/* variable shift costs */
  COSTS_N_BYTES (3),			/* constant shift costs */
  {COSTS_N_BYTES (3),			/* cost of starting multiply for QI */
   COSTS_N_BYTES (3),			/*				 HI */
   COSTS_N_BYTES (3),			/*				 SI */
   COSTS_N_BYTES (3),			/*				 DI */
   COSTS_N_BYTES (5)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_BYTES (3),			/* cost of a divide/mod for QI */
   COSTS_N_BYTES (3),			/*			    HI */
   COSTS_N_BYTES (3),			/*			    SI */
   COSTS_N_BYTES (3),			/*			    DI */
   COSTS_N_BYTES (5)},			/*			    other */
  COSTS_N_BYTES (3),			/* cost of movsx */
  COSTS_N_BYTES (3),			/* cost of movzx */
  0,					/* "large" insn */
  2,					/* MOVE_RATIO */
  2,				     /* cost for loading QImode using movzbl */
  {2, 2, 2},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {2, 2, 2},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {2, 2, 2},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {2, 2, 2},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  3,					/* cost of moving MMX register */
  {3, 3},				/* cost of loading MMX registers
					   in SImode and DImode */
  {3, 3},				/* cost of storing MMX registers
					   in SImode and DImode */
  3,					/* cost of moving SSE register */
  {3, 3, 3},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {3, 3, 3},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
  0,					/* size of l1 cache  */
  0,					/* size of l2 cache  */
  0,					/* size of prefetch block */
  0,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_BYTES (2),			/* cost of FADD and FSUB insns.  */
  COSTS_N_BYTES (2),			/* cost of FMUL instruction.  */
  COSTS_N_BYTES (2),			/* cost of FDIV instruction.  */
  COSTS_N_BYTES (2),			/* cost of FABS instruction.  */
  COSTS_N_BYTES (2),			/* cost of FCHS instruction.  */
  COSTS_N_BYTES (2),			/* cost of FSQRT instruction.  */
  ix86_size_memcpy,
  ix86_size_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  1,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  1,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

/* Processor costs (relative to an add) */
static stringop_algs i386_memcpy[2] = {
  {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
  DUMMY_STRINGOP_ALGS};
static stringop_algs i386_memset[2] = {
  {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
  DUMMY_STRINGOP_ALGS};

static const
struct processor_costs i386_cost = {	/* 386 specific costs */
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (3),			/* variable shift costs */
  COSTS_N_INSNS (2),			/* constant shift costs */
  {COSTS_N_INSNS (6),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (6),			/*				 HI */
   COSTS_N_INSNS (6),			/*				 SI */
   COSTS_N_INSNS (6),			/*				 DI */
   COSTS_N_INSNS (6)},			/*			      other */
  COSTS_N_INSNS (1),			/* cost of multiply per each bit set */
  {COSTS_N_INSNS (23),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (23),			/*			    HI */
   COSTS_N_INSNS (23),			/*			    SI */
   COSTS_N_INSNS (23),			/*			    DI */
   COSTS_N_INSNS (23)},			/*			    other */
  COSTS_N_INSNS (3),			/* cost of movsx */
  COSTS_N_INSNS (2),			/* cost of movzx */
  15,					/* "large" insn */
  3,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {2, 4, 2},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {2, 4, 2},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {8, 8, 8},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {8, 8, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {4, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 8, 16},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 8, 16},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
  0,					/* size of l1 cache  */
  0,					/* size of l2 cache  */
  0,					/* size of prefetch block */
  0,					/* number of parallel prefetches */
  1,					/* Branch cost */
  COSTS_N_INSNS (23),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (27),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (88),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (22),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (24),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (122),			/* cost of FSQRT instruction.  */
  i386_memcpy,
  i386_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs i486_memcpy[2] = {
  {rep_prefix_4_byte, {{-1, rep_prefix_4_byte, false}}},
  DUMMY_STRINGOP_ALGS};
static stringop_algs i486_memset[2] = {
  {rep_prefix_4_byte, {{-1, rep_prefix_4_byte, false}}},
  DUMMY_STRINGOP_ALGS};

static const
struct processor_costs i486_cost = {	/* 486 specific costs */
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (3),			/* variable shift costs */
  COSTS_N_INSNS (2),			/* constant shift costs */
  {COSTS_N_INSNS (12),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (12),			/*				 HI */
   COSTS_N_INSNS (12),			/*				 SI */
   COSTS_N_INSNS (12),			/*				 DI */
   COSTS_N_INSNS (12)},			/*			      other */
  1,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (40),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (40),			/*			    HI */
   COSTS_N_INSNS (40),			/*			    SI */
   COSTS_N_INSNS (40),			/*			    DI */
   COSTS_N_INSNS (40)},			/*			    other */
  COSTS_N_INSNS (3),			/* cost of movsx */
  COSTS_N_INSNS (2),			/* cost of movzx */
  15,					/* "large" insn */
  3,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {2, 4, 2},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {2, 4, 2},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {8, 8, 8},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {8, 8, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {4, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 8, 16},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 8, 16},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
  4,					/* size of l1 cache.  486 has 8kB cache
					   shared for code and data, so 4kB is
					   not really precise.  */
  4,					/* size of l2 cache  */
  0,					/* size of prefetch block */
  0,					/* number of parallel prefetches */
  1,					/* Branch cost */
  COSTS_N_INSNS (8),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (16),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (73),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (3),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (3),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (83),			/* cost of FSQRT instruction.  */
  i486_memcpy,
  i486_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs pentium_memcpy[2] = {
  {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};
static stringop_algs pentium_memset[2] = {
  {libcall, {{-1, rep_prefix_4_byte, false}}},
  DUMMY_STRINGOP_ALGS};

static const
struct processor_costs pentium_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (4),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (11),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (11),			/*				 HI */
   COSTS_N_INSNS (11),			/*				 SI */
   COSTS_N_INSNS (11),			/*				 DI */
   COSTS_N_INSNS (11)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (25),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (25),			/*			    HI */
   COSTS_N_INSNS (25),			/*			    SI */
   COSTS_N_INSNS (25),			/*			    DI */
   COSTS_N_INSNS (25)},			/*			    other */
  COSTS_N_INSNS (3),			/* cost of movsx */
  COSTS_N_INSNS (2),			/* cost of movzx */
  8,					/* "large" insn */
  6,					/* MOVE_RATIO */
  6,				     /* cost for loading QImode using movzbl */
  {2, 4, 2},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {2, 4, 2},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {2, 2, 6},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {4, 4, 6},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  8,					/* cost of moving MMX register */
  {8, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {8, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 8, 16},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 8, 16},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
  8,					/* size of l1 cache.  */
  8,					/* size of l2 cache  */
  0,					/* size of prefetch block */
  0,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (3),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (3),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (39),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (70),			/* cost of FSQRT instruction.  */
  pentium_memcpy,
  pentium_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static const
struct processor_costs lakemont_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1) + 1,		/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (11),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (11),			/*				 HI */
   COSTS_N_INSNS (11),			/*				 SI */
   COSTS_N_INSNS (11),			/*				 DI */
   COSTS_N_INSNS (11)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (25),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (25),			/*			    HI */
   COSTS_N_INSNS (25),			/*			    SI */
   COSTS_N_INSNS (25),			/*			    DI */
   COSTS_N_INSNS (25)},			/*			    other */
  COSTS_N_INSNS (3),			/* cost of movsx */
  COSTS_N_INSNS (2),			/* cost of movzx */
  8,					/* "large" insn */
  17,					/* MOVE_RATIO */
  6,				     /* cost for loading QImode using movzbl */
  {2, 4, 2},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {2, 4, 2},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {2, 2, 6},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {4, 4, 6},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  8,					/* cost of moving MMX register */
  {8, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {8, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 8, 16},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 8, 16},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
  8,					/* size of l1 cache.  */
  8,					/* size of l2 cache  */
  0,					/* size of prefetch block */
  0,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (3),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (3),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (39),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (70),			/* cost of FSQRT instruction.  */
  pentium_memcpy,
  pentium_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

/* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
   (we ensure the alignment).  For small blocks inline loop is still a
   noticeable win, for bigger blocks either rep movsl or rep movsb is
   way to go.  Rep movsb has apparently more expensive startup time in CPU,
   but after 4K the difference is down in the noise.  */
static stringop_algs pentiumpro_memcpy[2] = {
  {rep_prefix_4_byte, {{128, loop, false}, {1024, unrolled_loop, false},
                       {8192, rep_prefix_4_byte, false},
                       {-1, rep_prefix_1_byte, false}}},
  DUMMY_STRINGOP_ALGS};
static stringop_algs pentiumpro_memset[2] = {
  {rep_prefix_4_byte, {{1024, unrolled_loop, false},
                       {8192, rep_prefix_4_byte, false},
                       {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};
static const
struct processor_costs pentiumpro_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (4),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (4),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (4)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (17),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (17),			/*			    HI */
   COSTS_N_INSNS (17),			/*			    SI */
   COSTS_N_INSNS (17),			/*			    DI */
   COSTS_N_INSNS (17)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  6,					/* MOVE_RATIO */
  2,				     /* cost for loading QImode using movzbl */
  {4, 4, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {2, 2, 2},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {2, 2, 6},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {4, 4, 6},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {2, 2},				/* cost of loading MMX registers
					   in SImode and DImode */
  {2, 2},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {2, 2, 8},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {2, 2, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
  8,					/* size of l1 cache.  */
  256,					/* size of l2 cache  */
  32,					/* size of prefetch block */
  6,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (3),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (5),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (56),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (56),			/* cost of FSQRT instruction.  */
  pentiumpro_memcpy,
  pentiumpro_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs geode_memcpy[2] = {
  {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};
static stringop_algs geode_memset[2] = {
  {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};
static const
struct processor_costs geode_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (2),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (7),			/*				 SI */
   COSTS_N_INSNS (7),			/*				 DI */
   COSTS_N_INSNS (7)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (15),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (23),			/*			    HI */
   COSTS_N_INSNS (39),			/*			    SI */
   COSTS_N_INSNS (39),			/*			    DI */
   COSTS_N_INSNS (39)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  4,					/* MOVE_RATIO */
  1,				     /* cost for loading QImode using movzbl */
  {1, 1, 1},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {1, 1, 1},				/* cost of storing integer registers */
  1,					/* cost of reg,reg fld/fst */
  {1, 1, 1},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {4, 6, 6},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */

  2,					/* cost of moving MMX register */
  {2, 2},				/* cost of loading MMX registers
					   in SImode and DImode */
  {2, 2},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {2, 2, 8},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {2, 2, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
  64,					/* size of l1 cache.  */
  128,					/* size of l2 cache.  */
  32,					/* size of prefetch block */
  1,					/* number of parallel prefetches */
  1,					/* Branch cost */
  COSTS_N_INSNS (6),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (11),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (47),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (54),			/* cost of FSQRT instruction.  */
  geode_memcpy,
  geode_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs k6_memcpy[2] = {
  {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};
static stringop_algs k6_memset[2] = {
  {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};
static const
struct processor_costs k6_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (2),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (3),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (3),			/*				 DI */
   COSTS_N_INSNS (3)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (18),			/*			    HI */
   COSTS_N_INSNS (18),			/*			    SI */
   COSTS_N_INSNS (18),			/*			    DI */
   COSTS_N_INSNS (18)},			/*			    other */
  COSTS_N_INSNS (2),			/* cost of movsx */
  COSTS_N_INSNS (2),			/* cost of movzx */
  8,					/* "large" insn */
  4,					/* MOVE_RATIO */
  3,				     /* cost for loading QImode using movzbl */
  {4, 5, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {2, 3, 2},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {6, 6, 6},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {4, 4, 4},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {2, 2},				/* cost of loading MMX registers
					   in SImode and DImode */
  {2, 2},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {2, 2, 8},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {2, 2, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  6,					/* MMX or SSE register to integer */
  32,					/* size of l1 cache.  */
  32,					/* size of l2 cache.  Some models
					   have integrated l2 cache, but
					   optimizing for k6 is not important
					   enough to worry about that.  */
  32,					/* size of prefetch block */
  1,					/* number of parallel prefetches */
  1,					/* Branch cost */
  COSTS_N_INSNS (2),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (2),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (56),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (56),			/* cost of FSQRT instruction.  */
  k6_memcpy,
  k6_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

/* For some reason, Athlon deals better with REP prefix (relative to loops)
   compared to K8. Alignment becomes important after 8 bytes for memcpy and
   128 bytes for memset.  */
static stringop_algs athlon_memcpy[2] = {
  {libcall, {{2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};
static stringop_algs athlon_memset[2] = {
  {libcall, {{2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};
static const
struct processor_costs athlon_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (2),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (5),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (5),			/*				 HI */
   COSTS_N_INSNS (5),			/*				 SI */
   COSTS_N_INSNS (5),			/*				 DI */
   COSTS_N_INSNS (5)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (26),			/*			    HI */
   COSTS_N_INSNS (42),			/*			    SI */
   COSTS_N_INSNS (74),			/*			    DI */
   COSTS_N_INSNS (74)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {3, 4, 3},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {3, 4, 3},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {4, 4, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {4, 4},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 4, 6},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 5},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  5,					/* MMX or SSE register to integer */
  64,					/* size of l1 cache.  */
  256,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  6,					/* number of parallel prefetches */
  5,					/* Branch cost */
  COSTS_N_INSNS (4),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (4),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (24),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */
  athlon_memcpy,
  athlon_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

/* K8 has optimized REP instruction for medium sized blocks, but for very
   small blocks it is better to use loop. For large blocks, libcall can
   do nontemporary accesses and beat inline considerably.  */
static stringop_algs k8_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
             {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs k8_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false},
             {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
static const
struct processor_costs k8_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (2),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (5)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (26),			/*			    HI */
   COSTS_N_INSNS (42),			/*			    SI */
   COSTS_N_INSNS (74),			/*			    DI */
   COSTS_N_INSNS (74)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {3, 4, 3},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {3, 4, 3},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {4, 4, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {3, 3},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 3, 6},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 5},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  5,					/* MMX or SSE register to integer */
  64,					/* size of l1 cache.  */
  512,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  /* New AMD processors never drop prefetches; if they cannot be performed
     immediately, they are queued.  We set number of simultaneous prefetches
     to a large constant to reflect this (it probably is not a good idea not
     to limit number of prefetches at all, as their execution also takes some
     time).  */
  100,					/* number of parallel prefetches */
  3,					/* Branch cost */
  COSTS_N_INSNS (4),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (4),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (19),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */

  k8_memcpy,
  k8_memset,
  4,					/* scalar_stmt_cost.  */
  2,					/* scalar load_cost.  */
  2,					/* scalar_store_cost.  */
  5,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  2,					/* vec_align_load_cost.  */
  3,					/* vec_unalign_load_cost.  */
  3,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  2,					/* cond_not_taken_branch_cost.  */
};

/* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
   very small blocks it is better to use loop. For large blocks, libcall can
   do nontemporary accesses and beat inline considerably.  */
static stringop_algs amdfam10_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
             {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs amdfam10_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
struct processor_costs amdfam10_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (2),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (5)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (35),			/*			    HI */
   COSTS_N_INSNS (51),			/*			    SI */
   COSTS_N_INSNS (83),			/*			    DI */
   COSTS_N_INSNS (83)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {3, 4, 3},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {3, 4, 3},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {4, 4, 12},				/* cost of loading fp registers
		   			   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
 		   			   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {3, 3},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 4, 3},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 5},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
  					/* On K8:
  					    MOVD reg64, xmmreg Double FSTORE 4
					    MOVD reg32, xmmreg Double FSTORE 4
					   On AMDFAM10:
					    MOVD reg64, xmmreg Double FADD 3
							       1/1  1/1
					    MOVD reg32, xmmreg Double FADD 3
							       1/1  1/1 */
  64,					/* size of l1 cache.  */
  512,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  /* New AMD processors never drop prefetches; if they cannot be performed
     immediately, they are queued.  We set number of simultaneous prefetches
     to a large constant to reflect this (it probably is not a good idea not
     to limit number of prefetches at all, as their execution also takes some
     time).  */
  100,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (4),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (4),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (19),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */

  amdfam10_memcpy,
  amdfam10_memset,
  4,					/* scalar_stmt_cost.  */
  2,					/* scalar load_cost.  */
  2,					/* scalar_store_cost.  */
  6,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  2,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  2,					/* vec_store_cost.  */
  2,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

/*  BDVER1 has optimized REP instruction for medium sized blocks, but for
    very small blocks it is better to use loop. For large blocks, libcall
    can do nontemporary accesses and beat inline considerably.  */
static stringop_algs bdver1_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
             {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs bdver1_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};

const struct processor_costs bdver1_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (4),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (4),			/*				 SI */
   COSTS_N_INSNS (6),			/*				 DI */
   COSTS_N_INSNS (6)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (35),			/*			    HI */
   COSTS_N_INSNS (51),			/*			    SI */
   COSTS_N_INSNS (83),			/*			    DI */
   COSTS_N_INSNS (83)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {5, 5, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {5, 5, 12},				/* cost of loading fp registers
		   			   in SFmode, DFmode and XFmode */
  {4, 4, 8},				/* cost of storing fp registers
 		   			   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {4, 4},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 4, 4},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 4},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  2,					/* MMX or SSE register to integer */
  					/* On K8:
					    MOVD reg64, xmmreg Double FSTORE 4
					    MOVD reg32, xmmreg Double FSTORE 4
					   On AMDFAM10:
					    MOVD reg64, xmmreg Double FADD 3
							       1/1  1/1
					    MOVD reg32, xmmreg Double FADD 3
							       1/1  1/1 */
  16,					/* size of l1 cache.  */
  2048,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  /* New AMD processors never drop prefetches; if they cannot be performed
     immediately, they are queued.  We set number of simultaneous prefetches
     to a large constant to reflect this (it probably is not a good idea not
     to limit number of prefetches at all, as their execution also takes some
     time).  */
  100,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (6),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (6),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (42),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */

  bdver1_memcpy,
  bdver1_memset,
  6,					/* scalar_stmt_cost.  */
  4,					/* scalar load_cost.  */
  4,					/* scalar_store_cost.  */
  6,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  4,					/* vec_align_load_cost.  */
  4,					/* vec_unalign_load_cost.  */
  4,					/* vec_store_cost.  */
  4,					/* cond_taken_branch_cost.  */
  2,					/* cond_not_taken_branch_cost.  */
};

/*  BDVER2 has optimized REP instruction for medium sized blocks, but for
    very small blocks it is better to use loop. For large blocks, libcall
    can do nontemporary accesses and beat inline considerably.  */

static stringop_algs bdver2_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
             {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs bdver2_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};

const struct processor_costs bdver2_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (4),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (4),			/*				 SI */
   COSTS_N_INSNS (6),			/*				 DI */
   COSTS_N_INSNS (6)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (35),			/*			    HI */
   COSTS_N_INSNS (51),			/*			    SI */
   COSTS_N_INSNS (83),			/*			    DI */
   COSTS_N_INSNS (83)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {5, 5, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {5, 5, 12},				/* cost of loading fp registers
		   			   in SFmode, DFmode and XFmode */
  {4, 4, 8},				/* cost of storing fp registers
 		   			   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {4, 4},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 4, 4},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 4},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  2,					/* MMX or SSE register to integer */
  					/* On K8:
					    MOVD reg64, xmmreg Double FSTORE 4
					    MOVD reg32, xmmreg Double FSTORE 4
					   On AMDFAM10:
					    MOVD reg64, xmmreg Double FADD 3
							       1/1  1/1
					    MOVD reg32, xmmreg Double FADD 3
							       1/1  1/1 */
  16,					/* size of l1 cache.  */
  2048,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  /* New AMD processors never drop prefetches; if they cannot be performed
     immediately, they are queued.  We set number of simultaneous prefetches
     to a large constant to reflect this (it probably is not a good idea not
     to limit number of prefetches at all, as their execution also takes some
     time).  */
  100,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (6),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (6),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (42),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */

  bdver2_memcpy,
  bdver2_memset,
  6,					/* scalar_stmt_cost.  */
  4,					/* scalar load_cost.  */
  4,					/* scalar_store_cost.  */
  6,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  4,					/* vec_align_load_cost.  */
  4,					/* vec_unalign_load_cost.  */
  4,					/* vec_store_cost.  */
  4,					/* cond_taken_branch_cost.  */
  2,					/* cond_not_taken_branch_cost.  */
};


  /*  BDVER3 has optimized REP instruction for medium sized blocks, but for
      very small blocks it is better to use loop. For large blocks, libcall
      can do nontemporary accesses and beat inline considerably.  */
static stringop_algs bdver3_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
             {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs bdver3_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
struct processor_costs bdver3_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (4),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (4),			/*				 SI */
   COSTS_N_INSNS (6),			/*				 DI */
   COSTS_N_INSNS (6)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (35),			/*			    HI */
   COSTS_N_INSNS (51),			/*			    SI */
   COSTS_N_INSNS (83),			/*			    DI */
   COSTS_N_INSNS (83)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {5, 5, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {5, 5, 12},				/* cost of loading fp registers
		   			   in SFmode, DFmode and XFmode */
  {4, 4, 8},				/* cost of storing fp registers
 		   			   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {4, 4},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 4, 4},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 4},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  2,					/* MMX or SSE register to integer */
  16,					/* size of l1 cache.  */
  2048,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  /* New AMD processors never drop prefetches; if they cannot be performed
     immediately, they are queued.  We set number of simultaneous prefetches
     to a large constant to reflect this (it probably is not a good idea not
     to limit number of prefetches at all, as their execution also takes some
     time).  */
  100,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (6),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (6),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (42),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */

  bdver3_memcpy,
  bdver3_memset,
  6,					/* scalar_stmt_cost.  */
  4,					/* scalar load_cost.  */
  4,					/* scalar_store_cost.  */
  6,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  4,					/* vec_align_load_cost.  */
  4,					/* vec_unalign_load_cost.  */
  4,					/* vec_store_cost.  */
  4,					/* cond_taken_branch_cost.  */
  2,					/* cond_not_taken_branch_cost.  */
};

/*  BDVER4 has optimized REP instruction for medium sized blocks, but for
    very small blocks it is better to use loop. For large blocks, libcall
    can do nontemporary accesses and beat inline considerably.  */
static stringop_algs bdver4_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
             {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs bdver4_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
struct processor_costs bdver4_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (4),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (4),			/*				 SI */
   COSTS_N_INSNS (6),			/*				 DI */
   COSTS_N_INSNS (6)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (35),			/*			    HI */
   COSTS_N_INSNS (51),			/*			    SI */
   COSTS_N_INSNS (83),			/*			    DI */
   COSTS_N_INSNS (83)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {5, 5, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {5, 5, 12},				/* cost of loading fp registers
		   			   in SFmode, DFmode and XFmode */
  {4, 4, 8},				/* cost of storing fp registers
 		   			   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {4, 4},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 4, 4},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 4},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  2,					/* MMX or SSE register to integer */
  16,					/* size of l1 cache.  */
  2048,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  /* New AMD processors never drop prefetches; if they cannot be performed
     immediately, they are queued.  We set number of simultaneous prefetches
     to a large constant to reflect this (it probably is not a good idea not
     to limit number of prefetches at all, as their execution also takes some
     time).  */
  100,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (6),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (6),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (42),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */

  bdver4_memcpy,
  bdver4_memset,
  6,					/* scalar_stmt_cost.  */
  4,					/* scalar load_cost.  */
  4,					/* scalar_store_cost.  */
  6,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  4,					/* vec_align_load_cost.  */
  4,					/* vec_unalign_load_cost.  */
  4,					/* vec_store_cost.  */
  4,					/* cond_taken_branch_cost.  */
  2,					/* cond_not_taken_branch_cost.  */
};


/*  ZNVER1 has optimized REP instruction for medium sized blocks, but for
    very small blocks it is better to use loop.  For large blocks, libcall
    can do nontemporary accesses and beat inline considerably.  */
static stringop_algs znver1_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
	     {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
	     {-1, libcall, false}}}};
static stringop_algs znver1_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
	     {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
	     {-1, libcall, false}}}};
struct processor_costs znver1_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction.  */
  COSTS_N_INSNS (1),			/* cost of a lea instruction.  */
  COSTS_N_INSNS (1),			/* variable shift costs.  */
  COSTS_N_INSNS (1),			/* constant shift costs.  */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI.  */
   COSTS_N_INSNS (3),			/*				 HI.  */
   COSTS_N_INSNS (3),			/*				 SI.  */
   COSTS_N_INSNS (4),			/*				 DI.  */
   COSTS_N_INSNS (4)},			/*			      other.  */
  0,					/* cost of multiply per each bit
					    set.  */
  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI.  */
   COSTS_N_INSNS (35),			/*			    HI.  */
   COSTS_N_INSNS (51),			/*			    SI.  */
   COSTS_N_INSNS (83),			/*			    DI.  */
   COSTS_N_INSNS (83)},			/*			    other.  */
  COSTS_N_INSNS (1),			/* cost of movsx.  */
  COSTS_N_INSNS (1),			/* cost of movzx.  */
  8,					/* "large" insn.  */
  9,					/* MOVE_RATIO.  */
  4,					/* cost for loading QImode using
					   movzbl.  */
  {5, 5, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer
					   registers.  */
  2,					/* cost of reg,reg fld/fst.  */
  {5, 5, 12},				/* cost of loading fp registers
		   			   in SFmode, DFmode and XFmode.  */
  {4, 4, 8},				/* cost of storing fp registers
 		   			   in SFmode, DFmode and XFmode.  */
  2,					/* cost of moving MMX register.  */
  {4, 4},				/* cost of loading MMX registers
					   in SImode and DImode.  */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode.  */
  2,					/* cost of moving SSE register.  */
  {4, 4, 4},				/* cost of loading SSE registers
					   in SImode, DImode and TImode.  */
  {4, 4, 4},				/* cost of storing SSE registers
					   in SImode, DImode and TImode.  */
  2,					/* MMX or SSE register to integer.  */
  32,					/* size of l1 cache.  */
  512,					/* size of l2 cache.  */
  64,					/* size of prefetch block.  */
  /* New AMD processors never drop prefetches; if they cannot be performed
     immediately, they are queued.  We set number of simultaneous prefetches
     to a large constant to reflect this (it probably is not a good idea not
     to limit number of prefetches at all, as their execution also takes some
     time).  */
  100,					/* number of parallel prefetches.  */
  2,					/* Branch cost.  */
  COSTS_N_INSNS (6),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (6),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (42),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (52),			/* cost of FSQRT instruction.  */

  znver1_memcpy,
  znver1_memset,
  6,					/* scalar_stmt_cost.  */
  4,					/* scalar load_cost.  */
  4,					/* scalar_store_cost.  */
  6,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  4,					/* vec_align_load_cost.  */
  4,					/* vec_unalign_load_cost.  */
  4,					/* vec_store_cost.  */
  4,					/* cond_taken_branch_cost.  */
  2,					/* cond_not_taken_branch_cost.  */
};

  /* BTVER1 has optimized REP instruction for medium sized blocks, but for
     very small blocks it is better to use loop. For large blocks, libcall can
     do nontemporary accesses and beat inline considerably.  */
static stringop_algs btver1_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
             {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs btver1_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
const struct processor_costs btver1_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (2),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (5)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (35),			/*			    HI */
   COSTS_N_INSNS (51),			/*			    SI */
   COSTS_N_INSNS (83),			/*			    DI */
   COSTS_N_INSNS (83)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {3, 4, 3},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {3, 4, 3},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {4, 4, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {3, 3},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 4, 3},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 5},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
					/* On K8:
					   MOVD reg64, xmmreg Double FSTORE 4
					   MOVD reg32, xmmreg Double FSTORE 4
					   On AMDFAM10:
					   MOVD reg64, xmmreg Double FADD 3
							       1/1  1/1
					    MOVD reg32, xmmreg Double FADD 3
							       1/1  1/1 */
  32,					/* size of l1 cache.  */
  512,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  100,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (4),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (4),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (19),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */

  btver1_memcpy,
  btver1_memset,
  4,					/* scalar_stmt_cost.  */
  2,					/* scalar load_cost.  */
  2,					/* scalar_store_cost.  */
  6,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  2,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  2,					/* vec_store_cost.  */
  2,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs btver2_memcpy[2] = {
  {libcall, {{6, loop, false}, {14, unrolled_loop, false},
             {-1, rep_prefix_4_byte, false}}},
  {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs btver2_memset[2] = {
  {libcall, {{8, loop, false}, {24, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
const struct processor_costs btver2_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (2),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (5)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (19),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (35),			/*			    HI */
   COSTS_N_INSNS (51),			/*			    SI */
   COSTS_N_INSNS (83),			/*			    DI */
   COSTS_N_INSNS (83)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  9,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {3, 4, 3},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {3, 4, 3},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {4, 4, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {3, 3},				/* cost of loading MMX registers
					   in SImode and DImode */
  {4, 4},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {4, 4, 3},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {4, 4, 5},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  3,					/* MMX or SSE register to integer */
					/* On K8:
					   MOVD reg64, xmmreg Double FSTORE 4
					   MOVD reg32, xmmreg Double FSTORE 4
					   On AMDFAM10:
					   MOVD reg64, xmmreg Double FADD 3
							       1/1  1/1
					    MOVD reg32, xmmreg Double FADD 3
							       1/1  1/1 */
  32,					/* size of l1 cache.  */
  2048,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  100,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (4),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (4),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (19),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (35),			/* cost of FSQRT instruction.  */
  btver2_memcpy,
  btver2_memset,
  4,					/* scalar_stmt_cost.  */
  2,					/* scalar load_cost.  */
  2,					/* scalar_store_cost.  */
  6,					/* vec_stmt_cost.  */
  0,					/* vec_to_scalar_cost.  */
  2,					/* scalar_to_vec_cost.  */
  2,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  2,					/* vec_store_cost.  */
  2,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs pentium4_memcpy[2] = {
  {libcall, {{12, loop_1_byte, false}, {-1, rep_prefix_4_byte, false}}},
  DUMMY_STRINGOP_ALGS};
static stringop_algs pentium4_memset[2] = {
  {libcall, {{6, loop_1_byte, false}, {48, loop, false},
             {20480, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  DUMMY_STRINGOP_ALGS};

static const
struct processor_costs pentium4_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (3),			/* cost of a lea instruction */
  COSTS_N_INSNS (4),			/* variable shift costs */
  COSTS_N_INSNS (4),			/* constant shift costs */
  {COSTS_N_INSNS (15),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (15),			/*				 HI */
   COSTS_N_INSNS (15),			/*				 SI */
   COSTS_N_INSNS (15),			/*				 DI */
   COSTS_N_INSNS (15)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (56),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (56),			/*			    HI */
   COSTS_N_INSNS (56),			/*			    SI */
   COSTS_N_INSNS (56),			/*			    DI */
   COSTS_N_INSNS (56)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  16,					/* "large" insn */
  6,					/* MOVE_RATIO */
  2,				     /* cost for loading QImode using movzbl */
  {4, 5, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {2, 3, 2},				/* cost of storing integer registers */
  2,					/* cost of reg,reg fld/fst */
  {2, 2, 6},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {4, 4, 6},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {2, 2},				/* cost of loading MMX registers
					   in SImode and DImode */
  {2, 2},				/* cost of storing MMX registers
					   in SImode and DImode */
  12,					/* cost of moving SSE register */
  {12, 12, 12},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {2, 2, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  10,					/* MMX or SSE register to integer */
  8,					/* size of l1 cache.  */
  256,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  6,					/* number of parallel prefetches */
  2,					/* Branch cost */
  COSTS_N_INSNS (5),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (7),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (43),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (2),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (2),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (43),			/* cost of FSQRT instruction.  */
  pentium4_memcpy,
  pentium4_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs nocona_memcpy[2] = {
  {libcall, {{12, loop_1_byte, false}, {-1, rep_prefix_4_byte, false}}},
  {libcall, {{32, loop, false}, {20000, rep_prefix_8_byte, false},
             {100000, unrolled_loop, false}, {-1, libcall, false}}}};

static stringop_algs nocona_memset[2] = {
  {libcall, {{6, loop_1_byte, false}, {48, loop, false},
             {20480, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{24, loop, false}, {64, unrolled_loop, false},
             {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};

static const
struct processor_costs nocona_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1),			/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (10),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (10),			/*				 HI */
   COSTS_N_INSNS (10),			/*				 SI */
   COSTS_N_INSNS (10),			/*				 DI */
   COSTS_N_INSNS (10)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (66),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (66),			/*			    HI */
   COSTS_N_INSNS (66),			/*			    SI */
   COSTS_N_INSNS (66),			/*			    DI */
   COSTS_N_INSNS (66)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  16,					/* "large" insn */
  17,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {4, 4, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  3,					/* cost of reg,reg fld/fst */
  {12, 12, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {4, 4, 4},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  6,					/* cost of moving MMX register */
  {12, 12},				/* cost of loading MMX registers
					   in SImode and DImode */
  {12, 12},				/* cost of storing MMX registers
					   in SImode and DImode */
  6,					/* cost of moving SSE register */
  {12, 12, 12},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {12, 12, 12},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  8,					/* MMX or SSE register to integer */
  8,					/* size of l1 cache.  */
  1024,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  8,					/* number of parallel prefetches */
  1,					/* Branch cost */
  COSTS_N_INSNS (6),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (8),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (40),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (3),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (3),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (44),			/* cost of FSQRT instruction.  */
  nocona_memcpy,
  nocona_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs atom_memcpy[2] = {
  {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
  {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
             {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
static stringop_algs atom_memset[2] = {
  {libcall, {{8, loop, false}, {15, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{24, loop, false}, {32, unrolled_loop, false},
             {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
static const
struct processor_costs atom_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1) + 1,		/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (2)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (26),			/*			    HI */
   COSTS_N_INSNS (42),			/*			    SI */
   COSTS_N_INSNS (74),			/*			    DI */
   COSTS_N_INSNS (74)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  17,					/* MOVE_RATIO */
  4,					/* cost for loading QImode using movzbl */
  {4, 4, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {12, 12, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {8, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {8, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {8, 8, 8},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {8, 8, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  5,					/* MMX or SSE register to integer */
  32,					/* size of l1 cache.  */
  256,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  6,					/* number of parallel prefetches */
  3,					/* Branch cost */
  COSTS_N_INSNS (8),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (8),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (20),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
  atom_memcpy,
  atom_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs slm_memcpy[2] = {
  {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
  {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
             {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
static stringop_algs slm_memset[2] = {
  {libcall, {{8, loop, false}, {15, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{24, loop, false}, {32, unrolled_loop, false},
             {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
static const
struct processor_costs slm_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1) + 1,		/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (3),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (2)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (26),			/*			    HI */
   COSTS_N_INSNS (42),			/*			    SI */
   COSTS_N_INSNS (74),			/*			    DI */
   COSTS_N_INSNS (74)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  17,					/* MOVE_RATIO */
  4,					/* cost for loading QImode using movzbl */
  {4, 4, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {12, 12, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {8, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {8, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {8, 8, 8},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {8, 8, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  5,					/* MMX or SSE register to integer */
  32,					/* size of l1 cache.  */
  256,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  6,					/* number of parallel prefetches */
  3,					/* Branch cost */
  COSTS_N_INSNS (8),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (8),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (20),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
  slm_memcpy,
  slm_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  4,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

static stringop_algs intel_memcpy[2] = {
  {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
  {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
             {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
static stringop_algs intel_memset[2] = {
  {libcall, {{8, loop, false}, {15, unrolled_loop, false},
             {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
  {libcall, {{24, loop, false}, {32, unrolled_loop, false},
             {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
static const
struct processor_costs intel_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  COSTS_N_INSNS (1) + 1,		/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (3),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (2)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (26),			/*			    HI */
   COSTS_N_INSNS (42),			/*			    SI */
   COSTS_N_INSNS (74),			/*			    DI */
   COSTS_N_INSNS (74)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  17,					/* MOVE_RATIO */
  4,					/* cost for loading QImode using movzbl */
  {4, 4, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {12, 12, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {8, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {8, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {8, 8, 8},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {8, 8, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  5,					/* MMX or SSE register to integer */
  32,					/* size of l1 cache.  */
  256,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  6,					/* number of parallel prefetches */
  3,					/* Branch cost */
  COSTS_N_INSNS (8),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (8),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (20),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
  intel_memcpy,
  intel_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  4,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

/* Generic should produce code tuned for Core-i7 (and newer chips)
   and btver1 (and newer chips).  */

static stringop_algs generic_memcpy[2] = {
  {libcall, {{32, loop, false}, {8192, rep_prefix_4_byte, false},
             {-1, libcall, false}}},
  {libcall, {{32, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static stringop_algs generic_memset[2] = {
  {libcall, {{32, loop, false}, {8192, rep_prefix_4_byte, false},
             {-1, libcall, false}}},
  {libcall, {{32, loop, false}, {8192, rep_prefix_8_byte, false},
             {-1, libcall, false}}}};
static const
struct processor_costs generic_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  /* On all chips taken into consideration lea is 2 cycles and more.  With
     this cost however our current implementation of synth_mult results in
     use of unnecessary temporary registers causing regression on several
     SPECfp benchmarks.  */
  COSTS_N_INSNS (1) + 1,		/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (2)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (26),			/*			    HI */
   COSTS_N_INSNS (42),			/*			    SI */
   COSTS_N_INSNS (74),			/*			    DI */
   COSTS_N_INSNS (74)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  17,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {4, 4, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {12, 12, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {8, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {8, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {8, 8, 8},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {8, 8, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  5,					/* MMX or SSE register to integer */
  32,					/* size of l1 cache.  */
  512,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  6,					/* number of parallel prefetches */
  /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
     value is increased to perhaps more appropriate value of 5.  */
  3,					/* Branch cost */
  COSTS_N_INSNS (8),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (8),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (20),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
  generic_memcpy,
  generic_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};

/* core_cost should produce code tuned for Core familly of CPUs.  */
static stringop_algs core_memcpy[2] = {
  {libcall, {{1024, rep_prefix_4_byte, true}, {-1, libcall, false}}},
  {libcall, {{24, loop, true}, {128, rep_prefix_8_byte, true},
             {-1, libcall, false}}}};
static stringop_algs core_memset[2] = {
  {libcall, {{6, loop_1_byte, true},
             {24, loop, true},
             {8192, rep_prefix_4_byte, true},
             {-1, libcall, false}}},
  {libcall, {{24, loop, true}, {512, rep_prefix_8_byte, true},
             {-1, libcall, false}}}};

static const
struct processor_costs core_cost = {
  COSTS_N_INSNS (1),			/* cost of an add instruction */
  /* On all chips taken into consideration lea is 2 cycles and more.  With
     this cost however our current implementation of synth_mult results in
     use of unnecessary temporary registers causing regression on several
     SPECfp benchmarks.  */
  COSTS_N_INSNS (1) + 1,		/* cost of a lea instruction */
  COSTS_N_INSNS (1),			/* variable shift costs */
  COSTS_N_INSNS (1),			/* constant shift costs */
  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
   COSTS_N_INSNS (4),			/*				 HI */
   COSTS_N_INSNS (3),			/*				 SI */
   COSTS_N_INSNS (4),			/*				 DI */
   COSTS_N_INSNS (2)},			/*			      other */
  0,					/* cost of multiply per each bit set */
  {COSTS_N_INSNS (18),			/* cost of a divide/mod for QI */
   COSTS_N_INSNS (26),			/*			    HI */
   COSTS_N_INSNS (42),			/*			    SI */
   COSTS_N_INSNS (74),			/*			    DI */
   COSTS_N_INSNS (74)},			/*			    other */
  COSTS_N_INSNS (1),			/* cost of movsx */
  COSTS_N_INSNS (1),			/* cost of movzx */
  8,					/* "large" insn */
  17,					/* MOVE_RATIO */
  4,				     /* cost for loading QImode using movzbl */
  {4, 4, 4},				/* cost of loading integer registers
					   in QImode, HImode and SImode.
					   Relative to reg-reg move (2).  */
  {4, 4, 4},				/* cost of storing integer registers */
  4,					/* cost of reg,reg fld/fst */
  {12, 12, 12},				/* cost of loading fp registers
					   in SFmode, DFmode and XFmode */
  {6, 6, 8},				/* cost of storing fp registers
					   in SFmode, DFmode and XFmode */
  2,					/* cost of moving MMX register */
  {8, 8},				/* cost of loading MMX registers
					   in SImode and DImode */
  {8, 8},				/* cost of storing MMX registers
					   in SImode and DImode */
  2,					/* cost of moving SSE register */
  {8, 8, 8},				/* cost of loading SSE registers
					   in SImode, DImode and TImode */
  {8, 8, 8},				/* cost of storing SSE registers
					   in SImode, DImode and TImode */
  5,					/* MMX or SSE register to integer */
  64,					/* size of l1 cache.  */
  512,					/* size of l2 cache.  */
  64,					/* size of prefetch block */
  6,					/* number of parallel prefetches */
  /* FIXME perhaps more appropriate value is 5.  */
  3,					/* Branch cost */
  COSTS_N_INSNS (8),			/* cost of FADD and FSUB insns.  */
  COSTS_N_INSNS (8),			/* cost of FMUL instruction.  */
  COSTS_N_INSNS (20),			/* cost of FDIV instruction.  */
  COSTS_N_INSNS (8),			/* cost of FABS instruction.  */
  COSTS_N_INSNS (8),			/* cost of FCHS instruction.  */
  COSTS_N_INSNS (40),			/* cost of FSQRT instruction.  */
  core_memcpy,
  core_memset,
  1,					/* scalar_stmt_cost.  */
  1,					/* scalar load_cost.  */
  1,					/* scalar_store_cost.  */
  1,					/* vec_stmt_cost.  */
  1,					/* vec_to_scalar_cost.  */
  1,					/* scalar_to_vec_cost.  */
  1,					/* vec_align_load_cost.  */
  2,					/* vec_unalign_load_cost.  */
  1,					/* vec_store_cost.  */
  3,					/* cond_taken_branch_cost.  */
  1,					/* cond_not_taken_branch_cost.  */
};


/* Set by -mtune.  */
const struct processor_costs *ix86_tune_cost = &pentium_cost;

/* Set by -mtune or -Os.  */
const struct processor_costs *ix86_cost = &pentium_cost;

/* Processor feature/optimization bitmasks.  */
#define m_386 (1U<<PROCESSOR_I386)
#define m_486 (1U<<PROCESSOR_I486)
#define m_PENT (1U<<PROCESSOR_PENTIUM)
#define m_LAKEMONT (1U<<PROCESSOR_LAKEMONT)
#define m_PPRO (1U<<PROCESSOR_PENTIUMPRO)
#define m_PENT4 (1U<<PROCESSOR_PENTIUM4)
#define m_NOCONA (1U<<PROCESSOR_NOCONA)
#define m_P4_NOCONA (m_PENT4 | m_NOCONA)
#define m_CORE2 (1U<<PROCESSOR_CORE2)
#define m_NEHALEM (1U<<PROCESSOR_NEHALEM)
#define m_SANDYBRIDGE (1U<<PROCESSOR_SANDYBRIDGE)
#define m_HASWELL (1U<<PROCESSOR_HASWELL)
#define m_CORE_ALL (m_CORE2 | m_NEHALEM  | m_SANDYBRIDGE | m_HASWELL)
#define m_BONNELL (1U<<PROCESSOR_BONNELL)
#define m_SILVERMONT (1U<<PROCESSOR_SILVERMONT)
#define m_KNL (1U<<PROCESSOR_KNL)
#define m_SKYLAKE_AVX512 (1U<<PROCESSOR_SKYLAKE_AVX512)
#define m_INTEL (1U<<PROCESSOR_INTEL)

#define m_GEODE (1U<<PROCESSOR_GEODE)
#define m_K6 (1U<<PROCESSOR_K6)
#define m_K6_GEODE (m_K6 | m_GEODE)
#define m_K8 (1U<<PROCESSOR_K8)
#define m_ATHLON (1U<<PROCESSOR_ATHLON)
#define m_ATHLON_K8 (m_K8 | m_ATHLON)
#define m_AMDFAM10 (1U<<PROCESSOR_AMDFAM10)
#define m_BDVER1 (1U<<PROCESSOR_BDVER1)
#define m_BDVER2 (1U<<PROCESSOR_BDVER2)
#define m_BDVER3 (1U<<PROCESSOR_BDVER3)
#define m_BDVER4 (1U<<PROCESSOR_BDVER4)
#define m_ZNVER1 (1U<<PROCESSOR_ZNVER1)
#define m_BTVER1 (1U<<PROCESSOR_BTVER1)
#define m_BTVER2 (1U<<PROCESSOR_BTVER2)
#define m_BDVER	(m_BDVER1 | m_BDVER2 | m_BDVER3 | m_BDVER4)
#define m_BTVER (m_BTVER1 | m_BTVER2)
#define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER \
			| m_ZNVER1)

#define m_GENERIC (1U<<PROCESSOR_GENERIC)

const char* ix86_tune_feature_names[X86_TUNE_LAST] = {
#undef DEF_TUNE
#define DEF_TUNE(tune, name, selector) name,
#include "x86-tune.def"
#undef DEF_TUNE
};

/* Feature tests against the various tunings.  */
unsigned char ix86_tune_features[X86_TUNE_LAST];

/* Feature tests against the various tunings used to create ix86_tune_features
   based on the processor mask.  */
static unsigned int initial_ix86_tune_features[X86_TUNE_LAST] = {
#undef DEF_TUNE
#define DEF_TUNE(tune, name, selector) selector,
#include "x86-tune.def"
#undef DEF_TUNE
};

/* Feature tests against the various architecture variations.  */
unsigned char ix86_arch_features[X86_ARCH_LAST];

/* Feature tests against the various architecture variations, used to create
   ix86_arch_features based on the processor mask.  */
static unsigned int initial_ix86_arch_features[X86_ARCH_LAST] = {
  /* X86_ARCH_CMOV: Conditional move was added for pentiumpro.  */
  ~(m_386 | m_486 | m_PENT | m_LAKEMONT | m_K6),

  /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486.  */
  ~m_386,

  /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
  ~(m_386 | m_486),

  /* X86_ARCH_XADD: Exchange and add was added for 80486.  */
  ~m_386,

  /* X86_ARCH_BSWAP: Byteswap was added for 80486.  */
  ~m_386,
};

/* In case the average insn count for single function invocation is
   lower than this constant, emit fast (but longer) prologue and
   epilogue code.  */
#define FAST_PROLOGUE_INSN_COUNT 20

/* Names for 8 (low), 8 (high), and 16-bit registers, respectively.  */
static const char *const qi_reg_name[] = QI_REGISTER_NAMES;
static const char *const qi_high_reg_name[] = QI_HIGH_REGISTER_NAMES;
static const char *const hi_reg_name[] = HI_REGISTER_NAMES;

/* Array of the smallest class containing reg number REGNO, indexed by
   REGNO.  Used by REGNO_REG_CLASS in i386.h.  */

enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
{
  /* ax, dx, cx, bx */
  AREG, DREG, CREG, BREG,
  /* si, di, bp, sp */
  SIREG, DIREG, NON_Q_REGS, NON_Q_REGS,
  /* FP registers */
  FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS,
  FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS,
  /* arg pointer */
  NON_Q_REGS,
  /* flags, fpsr, fpcr, frame */
  NO_REGS, NO_REGS, NO_REGS, NON_Q_REGS,
  /* SSE registers */
  SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
  SSE_REGS, SSE_REGS,
  /* MMX registers */
  MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
  MMX_REGS, MMX_REGS,
  /* REX registers */
  NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
  NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
  /* SSE REX registers */
  SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
  SSE_REGS, SSE_REGS,
  /* AVX-512 SSE registers */
  EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
  EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
  EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
  EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
  /* Mask registers.  */
  MASK_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS,
  MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS,
  /* MPX bound registers */
  BND_REGS, BND_REGS, BND_REGS, BND_REGS,
};

/* The "default" register map used in 32bit mode.  */

int const dbx_register_map[FIRST_PSEUDO_REGISTER] =
{
  0, 2, 1, 3, 6, 7, 4, 5,		/* general regs */
  12, 13, 14, 15, 16, 17, 18, 19,	/* fp regs */
  -1, -1, -1, -1, -1,			/* arg, flags, fpsr, fpcr, frame */
  21, 22, 23, 24, 25, 26, 27, 28,	/* SSE */
  29, 30, 31, 32, 33, 34, 35, 36,       /* MMX */
  -1, -1, -1, -1, -1, -1, -1, -1,	/* extended integer registers */
  -1, -1, -1, -1, -1, -1, -1, -1,	/* extended SSE registers */
  -1, -1, -1, -1, -1, -1, -1, -1,       /* AVX-512 registers 16-23*/
  -1, -1, -1, -1, -1, -1, -1, -1,       /* AVX-512 registers 24-31*/
  93, 94, 95, 96, 97, 98, 99, 100,      /* Mask registers */
  101, 102, 103, 104,			/* bound registers */
};

/* The "default" register map used in 64bit mode.  */

int const dbx64_register_map[FIRST_PSEUDO_REGISTER] =
{
  0, 1, 2, 3, 4, 5, 6, 7,		/* general regs */
  33, 34, 35, 36, 37, 38, 39, 40,	/* fp regs */
  -1, -1, -1, -1, -1,			/* arg, flags, fpsr, fpcr, frame */
  17, 18, 19, 20, 21, 22, 23, 24,	/* SSE */
  41, 42, 43, 44, 45, 46, 47, 48,       /* MMX */
  8,9,10,11,12,13,14,15,		/* extended integer registers */
  25, 26, 27, 28, 29, 30, 31, 32,	/* extended SSE registers */
  67, 68, 69, 70, 71, 72, 73, 74,       /* AVX-512 registers 16-23 */
  75, 76, 77, 78, 79, 80, 81, 82,       /* AVX-512 registers 24-31 */
  118, 119, 120, 121, 122, 123, 124, 125, /* Mask registers */
  126, 127, 128, 129,			/* bound registers */
};

/* Define the register numbers to be used in Dwarf debugging information.
   The SVR4 reference port C compiler uses the following register numbers
   in its Dwarf output code:
	0 for %eax (gcc regno = 0)
	1 for %ecx (gcc regno = 2)
	2 for %edx (gcc regno = 1)
	3 for %ebx (gcc regno = 3)
	4 for %esp (gcc regno = 7)
	5 for %ebp (gcc regno = 6)
	6 for %esi (gcc regno = 4)
	7 for %edi (gcc regno = 5)
   The following three DWARF register numbers are never generated by
   the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
   believes these numbers have these meanings.
	8  for %eip    (no gcc equivalent)
	9  for %eflags (gcc regno = 17)
	10 for %trapno (no gcc equivalent)
   It is not at all clear how we should number the FP stack registers
   for the x86 architecture.  If the version of SDB on x86/svr4 were
   a bit less brain dead with respect to floating-point then we would
   have a precedent to follow with respect to DWARF register numbers
   for x86 FP registers, but the SDB on x86/svr4 is so completely
   broken with respect to FP registers that it is hardly worth thinking
   of it as something to strive for compatibility with.
   The version of x86/svr4 SDB I have at the moment does (partially)
   seem to believe that DWARF register number 11 is associated with
   the x86 register %st(0), but that's about all.  Higher DWARF
   register numbers don't seem to be associated with anything in
   particular, and even for DWARF regno 11, SDB only seems to under-
   stand that it should say that a variable lives in %st(0) (when
   asked via an `=' command) if we said it was in DWARF regno 11,
   but SDB still prints garbage when asked for the value of the
   variable in question (via a `/' command).
   (Also note that the labels SDB prints for various FP stack regs
   when doing an `x' command are all wrong.)
   Note that these problems generally don't affect the native SVR4
   C compiler because it doesn't allow the use of -O with -g and
   because when it is *not* optimizing, it allocates a memory
   location for each floating-point variable, and the memory
   location is what gets described in the DWARF AT_location
   attribute for the variable in question.
   Regardless of the severe mental illness of the x86/svr4 SDB, we
   do something sensible here and we use the following DWARF
   register numbers.  Note that these are all stack-top-relative
   numbers.
	11 for %st(0) (gcc regno = 8)
	12 for %st(1) (gcc regno = 9)
	13 for %st(2) (gcc regno = 10)
	14 for %st(3) (gcc regno = 11)
	15 for %st(4) (gcc regno = 12)
	16 for %st(5) (gcc regno = 13)
	17 for %st(6) (gcc regno = 14)
	18 for %st(7) (gcc regno = 15)
*/
int const svr4_dbx_register_map[FIRST_PSEUDO_REGISTER] =
{
  0, 2, 1, 3, 6, 7, 5, 4,		/* general regs */
  11, 12, 13, 14, 15, 16, 17, 18,	/* fp regs */
  -1, 9, -1, -1, -1,			/* arg, flags, fpsr, fpcr, frame */
  21, 22, 23, 24, 25, 26, 27, 28,	/* SSE registers */
  29, 30, 31, 32, 33, 34, 35, 36,	/* MMX registers */
  -1, -1, -1, -1, -1, -1, -1, -1,	/* extended integer registers */
  -1, -1, -1, -1, -1, -1, -1, -1,	/* extended SSE registers */
  -1, -1, -1, -1, -1, -1, -1, -1,       /* AVX-512 registers 16-23*/
  -1, -1, -1, -1, -1, -1, -1, -1,       /* AVX-512 registers 24-31*/
  93, 94, 95, 96, 97, 98, 99, 100,      /* Mask registers */
  101, 102, 103, 104,			/* bound registers */
};

/* Define parameter passing and return registers.  */

static int const x86_64_int_parameter_registers[6] =
{
  DI_REG, SI_REG, DX_REG, CX_REG, R8_REG, R9_REG
};

static int const x86_64_ms_abi_int_parameter_registers[4] =
{
  CX_REG, DX_REG, R8_REG, R9_REG
};

static int const x86_64_int_return_registers[4] =
{
  AX_REG, DX_REG, DI_REG, SI_REG
};

/* Additional registers that are clobbered by SYSV calls.  */

int const x86_64_ms_sysv_extra_clobbered_registers[12] =
{
  SI_REG, DI_REG,
  XMM6_REG, XMM7_REG,
  XMM8_REG, XMM9_REG, XMM10_REG, XMM11_REG,
  XMM12_REG, XMM13_REG, XMM14_REG, XMM15_REG
};

/* Define the structure for the machine field in struct function.  */

struct GTY(()) stack_local_entry {
  unsigned short mode;
  unsigned short n;
  rtx rtl;
  struct stack_local_entry *next;
};

/* Structure describing stack frame layout.
   Stack grows downward:

   [arguments]
					<- ARG_POINTER
   saved pc

   saved static chain			if ix86_static_chain_on_stack

   saved frame pointer			if frame_pointer_needed
					<- HARD_FRAME_POINTER
   [saved regs]
					<- regs_save_offset
   [padding0]

   [saved SSE regs]
					<- sse_regs_save_offset
   [padding1]          |
		       |		<- FRAME_POINTER
   [va_arg registers]  |
		       |
   [frame]	       |
		       |
   [padding2]	       | = to_allocate
					<- STACK_POINTER
  */
struct ix86_frame
{
  int nsseregs;
  int nregs;
  int va_arg_size;
  int red_zone_size;
  int outgoing_arguments_size;

  /* The offsets relative to ARG_POINTER.  */
  HOST_WIDE_INT frame_pointer_offset;
  HOST_WIDE_INT hard_frame_pointer_offset;
  HOST_WIDE_INT stack_pointer_offset;
  HOST_WIDE_INT hfp_save_offset;
  HOST_WIDE_INT reg_save_offset;
  HOST_WIDE_INT sse_reg_save_offset;

  /* When save_regs_using_mov is set, emit prologue using
     move instead of push instructions.  */
  bool save_regs_using_mov;
};

/* Which cpu are we scheduling for.  */
enum attr_cpu ix86_schedule;

/* Which cpu are we optimizing for.  */
enum processor_type ix86_tune;

/* Which instruction set architecture to use.  */
enum processor_type ix86_arch;

/* True if processor has SSE prefetch instruction.  */
unsigned char x86_prefetch_sse;

/* -mstackrealign option */
static const char ix86_force_align_arg_pointer_string[]
  = "force_align_arg_pointer";

static rtx (*ix86_gen_leave) (void);
static rtx (*ix86_gen_add3) (rtx, rtx, rtx);
static rtx (*ix86_gen_sub3) (rtx, rtx, rtx);
static rtx (*ix86_gen_sub3_carry) (rtx, rtx, rtx, rtx, rtx);
static rtx (*ix86_gen_one_cmpl2) (rtx, rtx);
static rtx (*ix86_gen_monitor) (rtx, rtx, rtx);
static rtx (*ix86_gen_monitorx) (rtx, rtx, rtx);
static rtx (*ix86_gen_clzero) (rtx);
static rtx (*ix86_gen_andsp) (rtx, rtx, rtx);
static rtx (*ix86_gen_allocate_stack_worker) (rtx, rtx);
static rtx (*ix86_gen_adjust_stack_and_probe) (rtx, rtx, rtx);
static rtx (*ix86_gen_probe_stack_range) (rtx, rtx, rtx);
static rtx (*ix86_gen_tls_global_dynamic_64) (rtx, rtx, rtx);
static rtx (*ix86_gen_tls_local_dynamic_base_64) (rtx, rtx);

/* Preferred alignment for stack boundary in bits.  */
unsigned int ix86_preferred_stack_boundary;

/* Alignment for incoming stack boundary in bits specified at
   command line.  */
static unsigned int ix86_user_incoming_stack_boundary;

/* Default alignment for incoming stack boundary in bits.  */
static unsigned int ix86_default_incoming_stack_boundary;

/* Alignment for incoming stack boundary in bits.  */
unsigned int ix86_incoming_stack_boundary;

/* Calling abi specific va_list type nodes.  */
static GTY(()) tree sysv_va_list_type_node;
static GTY(()) tree ms_va_list_type_node;

/* Prefix built by ASM_GENERATE_INTERNAL_LABEL.  */
char internal_label_prefix[16];
int internal_label_prefix_len;

/* Fence to use after loop using movnt.  */
tree x86_mfence;

/* Register class used for passing given 64bit part of the argument.
   These represent classes as documented by the PS ABI, with the exception
   of SSESF, SSEDF classes, that are basically SSE class, just gcc will
   use SF or DFmode move instead of DImode to avoid reformatting penalties.

   Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
   whenever possible (upper half does contain padding).  */
enum x86_64_reg_class
  {
    X86_64_NO_CLASS,
    X86_64_INTEGER_CLASS,
    X86_64_INTEGERSI_CLASS,
    X86_64_SSE_CLASS,
    X86_64_SSESF_CLASS,
    X86_64_SSEDF_CLASS,
    X86_64_SSEUP_CLASS,
    X86_64_X87_CLASS,
    X86_64_X87UP_CLASS,
    X86_64_COMPLEX_X87_CLASS,
    X86_64_MEMORY_CLASS
  };

#define MAX_CLASSES 8

/* Table of constants used by fldpi, fldln2, etc....  */
static REAL_VALUE_TYPE ext_80387_constants_table [5];
static bool ext_80387_constants_init = 0;


static struct machine_function * ix86_init_machine_status (void);
static rtx ix86_function_value (const_tree, const_tree, bool);
static bool ix86_function_value_regno_p (const unsigned int);
static unsigned int ix86_function_arg_boundary (machine_mode,
						const_tree);
static rtx ix86_static_chain (const_tree, bool);
static int ix86_function_regparm (const_tree, const_tree);
static void ix86_compute_frame_layout (struct ix86_frame *);
static bool ix86_expand_vector_init_one_nonzero (bool, machine_mode,
						 rtx, rtx, int);
static void ix86_add_new_builtins (HOST_WIDE_INT, HOST_WIDE_INT);
static tree ix86_canonical_va_list_type (tree);
static void predict_jump (int);
static unsigned int split_stack_prologue_scratch_regno (void);
static bool i386_asm_output_addr_const_extra (FILE *, rtx);

enum ix86_function_specific_strings
{
  IX86_FUNCTION_SPECIFIC_ARCH,
  IX86_FUNCTION_SPECIFIC_TUNE,
  IX86_FUNCTION_SPECIFIC_MAX
};

static char *ix86_target_string (HOST_WIDE_INT, HOST_WIDE_INT, int, int,
				 const char *, const char *, enum fpmath_unit,
				 bool);
static void ix86_function_specific_save (struct cl_target_option *,
					 struct gcc_options *opts);
static void ix86_function_specific_restore (struct gcc_options *opts,
					    struct cl_target_option *);
static void ix86_function_specific_post_stream_in (struct cl_target_option *);
static void ix86_function_specific_print (FILE *, int,
					  struct cl_target_option *);
static bool ix86_valid_target_attribute_p (tree, tree, tree, int);
static bool ix86_valid_target_attribute_inner_p (tree, char *[],
						 struct gcc_options *,
						 struct gcc_options *,
						 struct gcc_options *);
static bool ix86_can_inline_p (tree, tree);
static void ix86_set_current_function (tree);
static unsigned int ix86_minimum_incoming_stack_boundary (bool);

static enum calling_abi ix86_function_abi (const_tree);


#ifndef SUBTARGET32_DEFAULT_CPU
#define SUBTARGET32_DEFAULT_CPU "i386"
#endif

/* Whether -mtune= or -march= were specified */
static int ix86_tune_defaulted;
static int ix86_arch_specified;

/* Vectorization library interface and handlers.  */
static tree (*ix86_veclib_handler) (combined_fn, tree, tree);

static tree ix86_veclibabi_svml (combined_fn, tree, tree);
static tree ix86_veclibabi_acml (combined_fn, tree, tree);

/* Processor target table, indexed by processor number */
struct ptt
{
  const char *const name;			/* processor name  */
  const struct processor_costs *cost;		/* Processor costs */
  const int align_loop;				/* Default alignments.  */
  const int align_loop_max_skip;
  const int align_jump;
  const int align_jump_max_skip;
  const int align_func;
};

/* This table must be in sync with enum processor_type in i386.h.  */ 
static const struct ptt processor_target_table[PROCESSOR_max] =
{
  {"generic", &generic_cost, 16, 10, 16, 10, 16},
  {"i386", &i386_cost, 4, 3, 4, 3, 4},
  {"i486", &i486_cost, 16, 15, 16, 15, 16},
  {"pentium", &pentium_cost, 16, 7, 16, 7, 16},
  {"lakemont", &lakemont_cost, 16, 7, 16, 7, 16},
  {"pentiumpro", &pentiumpro_cost, 16, 15, 16, 10, 16},
  {"pentium4", &pentium4_cost, 0, 0, 0, 0, 0},
  {"nocona", &nocona_cost, 0, 0, 0, 0, 0},
  {"core2", &core_cost, 16, 10, 16, 10, 16},
  {"nehalem", &core_cost, 16, 10, 16, 10, 16},
  {"sandybridge", &core_cost, 16, 10, 16, 10, 16},
  {"haswell", &core_cost, 16, 10, 16, 10, 16},
  {"bonnell", &atom_cost, 16, 15, 16, 7, 16},
  {"silvermont", &slm_cost, 16, 15, 16, 7, 16},
  {"knl", &slm_cost, 16, 15, 16, 7, 16},
  {"skylake-avx512", &core_cost, 16, 10, 16, 10, 16},
  {"intel", &intel_cost, 16, 15, 16, 7, 16},
  {"geode", &geode_cost, 0, 0, 0, 0, 0},
  {"k6", &k6_cost, 32, 7, 32, 7, 32},
  {"athlon", &athlon_cost, 16, 7, 16, 7, 16},
  {"k8", &k8_cost, 16, 7, 16, 7, 16},
  {"amdfam10", &amdfam10_cost, 32, 24, 32, 7, 32},
  {"bdver1", &bdver1_cost, 16, 10, 16, 7, 11},
  {"bdver2", &bdver2_cost, 16, 10, 16, 7, 11},
  {"bdver3", &bdver3_cost, 16, 10, 16, 7, 11},
  {"bdver4", &bdver4_cost, 16, 10, 16, 7, 11},
  {"btver1", &btver1_cost, 16, 10, 16, 7, 11},
  {"btver2", &btver2_cost, 16, 10, 16, 7, 11},
  {"znver1", &znver1_cost, 16, 15, 16, 15, 16}
};

static unsigned int
rest_of_handle_insert_vzeroupper (void)
{
  int i;

  /* vzeroupper instructions are inserted immediately after reload to
     account for possible spills from 256bit registers.  The pass
     reuses mode switching infrastructure by re-running mode insertion
     pass, so disable entities that have already been processed.  */
  for (i = 0; i < MAX_386_ENTITIES; i++)
    ix86_optimize_mode_switching[i] = 0;

  ix86_optimize_mode_switching[AVX_U128] = 1;

  /* Call optimize_mode_switching.  */
  g->get_passes ()->execute_pass_mode_switching ();
  return 0;
}

/* Return 1 if INSN uses or defines a hard register.
   Hard register uses in a memory address are ignored.
   Clobbers and flags definitions are ignored.  */

static bool
has_non_address_hard_reg (rtx_insn *insn)
{
  df_ref ref;
  FOR_EACH_INSN_DEF (ref, insn)
    if (HARD_REGISTER_P (DF_REF_REAL_REG (ref))
	&& !DF_REF_FLAGS_IS_SET (ref, DF_REF_MUST_CLOBBER)
	&& DF_REF_REGNO (ref) != FLAGS_REG)
      return true;

  FOR_EACH_INSN_USE (ref, insn)
    if (!DF_REF_REG_MEM_P (ref) && HARD_REGISTER_P (DF_REF_REAL_REG (ref)))
      return true;

  return false;
}

/* Check if comparison INSN may be transformed
   into vector comparison.  Currently we transform
   zero checks only which look like:

   (set (reg:CCZ 17 flags)
        (compare:CCZ (ior:SI (subreg:SI (reg:DI x) 4)
                             (subreg:SI (reg:DI x) 0))
		     (const_int 0 [0])))  */

static bool
convertible_comparison_p (rtx_insn *insn)
{
  if (!TARGET_SSE4_1)
    return false;

  rtx def_set = single_set (insn);

  gcc_assert (def_set);

  rtx src = SET_SRC (def_set);
  rtx dst = SET_DEST (def_set);

  gcc_assert (GET_CODE (src) == COMPARE);

  if (GET_CODE (dst) != REG
      || REGNO (dst) != FLAGS_REG
      || GET_MODE (dst) != CCZmode)
    return false;

  rtx op1 = XEXP (src, 0);
  rtx op2 = XEXP (src, 1);

  if (op2 != CONST0_RTX (GET_MODE (op2)))
    return false;

  if (GET_CODE (op1) != IOR)
    return false;

  op2 = XEXP (op1, 1);
  op1 = XEXP (op1, 0);

  if (!SUBREG_P (op1)
      || !SUBREG_P (op2)
      || GET_MODE (op1) != SImode
      || GET_MODE (op2) != SImode
      || ((SUBREG_BYTE (op1) != 0
	   || SUBREG_BYTE (op2) != GET_MODE_SIZE (SImode))
	  && (SUBREG_BYTE (op2) != 0
	      || SUBREG_BYTE (op1) != GET_MODE_SIZE (SImode))))
    return false;

  op1 = SUBREG_REG (op1);
  op2 = SUBREG_REG (op2);

  if (op1 != op2
      || !REG_P (op1)
      || GET_MODE (op1) != DImode)
    return false;

  return true;
}

/* The DImode version of scalar_to_vector_candidate_p.  */

static bool
dimode_scalar_to_vector_candidate_p (rtx_insn *insn)
{
  rtx def_set = single_set (insn);

  if (!def_set)
    return false;

  if (has_non_address_hard_reg (insn))
    return false;

  rtx src = SET_SRC (def_set);
  rtx dst = SET_DEST (def_set);

  if (GET_CODE (src) == COMPARE)
    return convertible_comparison_p (insn);

  /* We are interested in DImode promotion only.  */
  if ((GET_MODE (src) != DImode
       && !CONST_INT_P (src))
      || GET_MODE (dst) != DImode)
    return false;

  if (!REG_P (dst) && !MEM_P (dst))
    return false;

  switch (GET_CODE (src))
    {
    case ASHIFT:
    case LSHIFTRT:
      /* FIXME: consider also variable shifts.  */
      if (!CONST_INT_P (XEXP (src, 1))
	  || !IN_RANGE (INTVAL (XEXP (src, 1)), 0, 63))
	return false;
      break;

    case PLUS:
    case MINUS:
    case IOR:
    case XOR:
    case AND:
      if (!REG_P (XEXP (src, 1))
	  && !MEM_P (XEXP (src, 1))
	  && !CONST_INT_P (XEXP (src, 1)))
	return false;
      break;

    case NEG:
    case NOT:
      break;

    case REG:
      return true;

    case MEM:
    case CONST_INT:
      return REG_P (dst);

    default:
      return false;
    }

  if (!REG_P (XEXP (src, 0))
      && !MEM_P (XEXP (src, 0))
      && !CONST_INT_P (XEXP (src, 0))
      /* Check for andnot case.  */
      && (GET_CODE (src) != AND
	  || GET_CODE (XEXP (src, 0)) != NOT
	  || !REG_P (XEXP (XEXP (src, 0), 0))))
      return false;

  if ((GET_MODE (XEXP (src, 0)) != DImode
       && !CONST_INT_P (XEXP (src, 0)))
      || (GET_CODE (src) != NEG
	  && GET_CODE (src) != NOT
	  && GET_MODE (XEXP (src, 1)) != DImode
	  && !CONST_INT_P (XEXP (src, 1))))
    return false;

  return true;
}

/* The TImode version of scalar_to_vector_candidate_p.  */

static bool
timode_scalar_to_vector_candidate_p (rtx_insn *insn)
{
  rtx def_set = single_set (insn);

  if (!def_set)
    return false;

  if (has_non_address_hard_reg (insn))
    return false;

  rtx src = SET_SRC (def_set);
  rtx dst = SET_DEST (def_set);

  /* Only TImode load and store are allowed.  */
  if (GET_MODE (dst) != TImode)
    return false;

  if (MEM_P (dst))
    {
      /* Check for store.  Memory must be aligned or unaligned store
	 is optimal.  Only support store from register, standard SSE
	 constant or CONST_WIDE_INT generated from piecewise store.

	 ??? Verify performance impact before enabling CONST_INT for
	 __int128 store.  */
      if (misaligned_operand (dst, TImode)
	  && !TARGET_SSE_UNALIGNED_STORE_OPTIMAL)
	return false;

      switch (GET_CODE (src))
	{
	default:
	  return false;

	case REG:
	case CONST_WIDE_INT:
	  return true;

	case CONST_INT:
	  return standard_sse_constant_p (src, TImode);
	}
    }
  else if (MEM_P (src))
    {
      /* Check for load.  Memory must be aligned or unaligned load is
	 optimal.  */
      return (REG_P (dst)
	      && (!misaligned_operand (src, TImode)
		  || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL));
    }

  return false;
}

/* Return 1 if INSN may be converted into vector
   instruction.  */

static bool
scalar_to_vector_candidate_p (rtx_insn *insn)
{
  if (TARGET_64BIT)
    return timode_scalar_to_vector_candidate_p (insn);
  else
    return dimode_scalar_to_vector_candidate_p (insn);
}

/* The DImode version of remove_non_convertible_regs.  */

static void
dimode_remove_non_convertible_regs (bitmap candidates)
{
  bitmap_iterator bi;
  unsigned id;
  bitmap regs = BITMAP_ALLOC (NULL);

  EXECUTE_IF_SET_IN_BITMAP (candidates, 0, id, bi)
    {
      rtx def_set = single_set (DF_INSN_UID_GET (id)->insn);
      rtx reg = SET_DEST (def_set);

      if (!REG_P (reg)
	  || bitmap_bit_p (regs, REGNO (reg))
	  || HARD_REGISTER_P (reg))
	continue;

      for (df_ref def = DF_REG_DEF_CHAIN (REGNO (reg));
	   def;
	   def = DF_REF_NEXT_REG (def))
	{
	  if (!bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
	    {
	      if (dump_file)
		fprintf (dump_file,
			 "r%d has non convertible definition in insn %d\n",
			 REGNO (reg), DF_REF_INSN_UID (def));

	      bitmap_set_bit (regs, REGNO (reg));
	      break;
	    }
	}
    }

  EXECUTE_IF_SET_IN_BITMAP (regs, 0, id, bi)
    {
      for (df_ref def = DF_REG_DEF_CHAIN (id);
	   def;
	   def = DF_REF_NEXT_REG (def))
	if (bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
	  {
	    if (dump_file)
	      fprintf (dump_file, "Removing insn %d from candidates list\n",
		       DF_REF_INSN_UID (def));

	    bitmap_clear_bit (candidates, DF_REF_INSN_UID (def));
	  }
    }

  BITMAP_FREE (regs);
}

/* For a register REGNO, scan instructions for its defs and uses.
   Put REGNO in REGS if a def or use isn't in CANDIDATES.  */

static void
timode_check_non_convertible_regs (bitmap candidates, bitmap regs,
				   unsigned int regno)
{
  for (df_ref def = DF_REG_DEF_CHAIN (regno);
       def;
       def = DF_REF_NEXT_REG (def))
    {
      if (!bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
	{
	  if (dump_file)
	    fprintf (dump_file,
		     "r%d has non convertible def in insn %d\n",
		     regno, DF_REF_INSN_UID (def));

	  bitmap_set_bit (regs, regno);
	  break;
	}
    }

  for (df_ref ref = DF_REG_USE_CHAIN (regno);
       ref;
       ref = DF_REF_NEXT_REG (ref))
    {
      /* Debug instructions are skipped.  */
      if (NONDEBUG_INSN_P (DF_REF_INSN (ref))
	  && !bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
	{
	  if (dump_file)
	    fprintf (dump_file,
		     "r%d has non convertible use in insn %d\n",
		     regno, DF_REF_INSN_UID (ref));

	  bitmap_set_bit (regs, regno);
	  break;
	}
    }
}

/* The TImode version of remove_non_convertible_regs.  */

static void
timode_remove_non_convertible_regs (bitmap candidates)
{
  bitmap_iterator bi;
  unsigned id;
  bitmap regs = BITMAP_ALLOC (NULL);

  EXECUTE_IF_SET_IN_BITMAP (candidates, 0, id, bi)
    {
      rtx def_set = single_set (DF_INSN_UID_GET (id)->insn);
      rtx dest = SET_DEST (def_set);
      rtx src = SET_SRC (def_set);

      if ((!REG_P (dest)
	   || bitmap_bit_p (regs, REGNO (dest))
	   || HARD_REGISTER_P (dest))
	  && (!REG_P (src)
	      || bitmap_bit_p (regs, REGNO (src))
	      || HARD_REGISTER_P (src)))
	continue;

      if (REG_P (dest))
	timode_check_non_convertible_regs (candidates, regs,
					   REGNO (dest));

      if (REG_P (src))
	timode_check_non_convertible_regs (candidates, regs,
					   REGNO (src));
    }

  EXECUTE_IF_SET_IN_BITMAP (regs, 0, id, bi)
    {
      for (df_ref def = DF_REG_DEF_CHAIN (id);
	   def;
	   def = DF_REF_NEXT_REG (def))
	if (bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
	  {
	    if (dump_file)
	      fprintf (dump_file, "Removing insn %d from candidates list\n",
		       DF_REF_INSN_UID (def));

	    bitmap_clear_bit (candidates, DF_REF_INSN_UID (def));
	  }

      for (df_ref ref = DF_REG_USE_CHAIN (id);
	   ref;
	   ref = DF_REF_NEXT_REG (ref))
	if (bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
	  {
	    if (dump_file)
	      fprintf (dump_file, "Removing insn %d from candidates list\n",
		       DF_REF_INSN_UID (ref));

	    bitmap_clear_bit (candidates, DF_REF_INSN_UID (ref));
	  }
    }

  BITMAP_FREE (regs);
}

/* For a given bitmap of insn UIDs scans all instruction and
   remove insn from CANDIDATES in case it has both convertible
   and not convertible definitions.

   All insns in a bitmap are conversion candidates according to
   scalar_to_vector_candidate_p.  Currently it implies all insns
   are single_set.  */

static void
remove_non_convertible_regs (bitmap candidates)
{
  if (TARGET_64BIT)
    timode_remove_non_convertible_regs (candidates);
  else
    dimode_remove_non_convertible_regs (candidates);
}

class scalar_chain
{
 public:
  scalar_chain ();
  virtual ~scalar_chain ();

  static unsigned max_id;

  /* ID of a chain.  */
  unsigned int chain_id;
  /* A queue of instructions to be included into a chain.  */
  bitmap queue;
  /* Instructions included into a chain.  */
  bitmap insns;
  /* All registers defined by a chain.  */
  bitmap defs;
  /* Registers used in both vector and sclar modes.  */
  bitmap defs_conv;

  void build (bitmap candidates, unsigned insn_uid);
  virtual int compute_convert_gain () = 0;
  int convert ();

 protected:
  void add_to_queue (unsigned insn_uid);
  void emit_conversion_insns (rtx insns, rtx_insn *pos);

 private:
  void add_insn (bitmap candidates, unsigned insn_uid);
  void analyze_register_chain (bitmap candidates, df_ref ref);
  virtual void mark_dual_mode_def (df_ref def) = 0;
  virtual void convert_insn (rtx_insn *insn) = 0;
  virtual void convert_registers () = 0;
};

class dimode_scalar_chain : public scalar_chain
{
 public:
  int compute_convert_gain ();
 private:
  void mark_dual_mode_def (df_ref def);
  rtx replace_with_subreg (rtx x, rtx reg, rtx subreg);
  void replace_with_subreg_in_insn (rtx_insn *insn, rtx reg, rtx subreg);
  void convert_insn (rtx_insn *insn);
  void convert_op (rtx *op, rtx_insn *insn);
  void convert_reg (unsigned regno);
  void make_vector_copies (unsigned regno);
  void convert_registers ();
  int vector_const_cost (rtx exp);
};

class timode_scalar_chain : public scalar_chain
{
 public:
  /* Convert from TImode to V1TImode is always faster.  */
  int compute_convert_gain () { return 1; }

 private:
  void mark_dual_mode_def (df_ref def);
  void fix_debug_reg_uses (rtx reg);
  void convert_insn (rtx_insn *insn);
  /* We don't convert registers to difference size.  */
  void convert_registers () {}
};

unsigned scalar_chain::max_id = 0;

/* Initialize new chain.  */

scalar_chain::scalar_chain ()
{
  chain_id = ++max_id;

   if (dump_file)
    fprintf (dump_file, "Created a new instruction chain #%d\n", chain_id);

  bitmap_obstack_initialize (NULL);
  insns = BITMAP_ALLOC (NULL);
  defs = BITMAP_ALLOC (NULL);
  defs_conv = BITMAP_ALLOC (NULL);
  queue = NULL;
}

/* Free chain's data.  */

scalar_chain::~scalar_chain ()
{
  BITMAP_FREE (insns);
  BITMAP_FREE (defs);
  BITMAP_FREE (defs_conv);
  bitmap_obstack_release (NULL);
}

/* Add instruction into chains' queue.  */

void
scalar_chain::add_to_queue (unsigned insn_uid)
{
  if (bitmap_bit_p (insns, insn_uid)
      || bitmap_bit_p (queue, insn_uid))
    return;

  if (dump_file)
    fprintf (dump_file, "  Adding insn %d into chain's #%d queue\n",
	     insn_uid, chain_id);
  bitmap_set_bit (queue, insn_uid);
}

/* For DImode conversion, mark register defined by DEF as requiring
   conversion.  */

void
dimode_scalar_chain::mark_dual_mode_def (df_ref def)
{
  gcc_assert (DF_REF_REG_DEF_P (def));

  if (bitmap_bit_p (defs_conv, DF_REF_REGNO (def)))
    return;

  if (dump_file)
    fprintf (dump_file,
	     "  Mark r%d def in insn %d as requiring both modes in chain #%d\n",
	     DF_REF_REGNO (def), DF_REF_INSN_UID (def), chain_id);

  bitmap_set_bit (defs_conv, DF_REF_REGNO (def));
}

/* For TImode conversion, it is unused.  */

void
timode_scalar_chain::mark_dual_mode_def (df_ref)
{
  gcc_unreachable ();
}

/* Check REF's chain to add new insns into a queue
   and find registers requiring conversion.  */

void
scalar_chain::analyze_register_chain (bitmap candidates, df_ref ref)
{
  df_link *chain;

  gcc_assert (bitmap_bit_p (insns, DF_REF_INSN_UID (ref))
	      || bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)));
  add_to_queue (DF_REF_INSN_UID (ref));

  for (chain = DF_REF_CHAIN (ref); chain; chain = chain->next)
    {
      unsigned uid = DF_REF_INSN_UID (chain->ref);

      if (!NONDEBUG_INSN_P (DF_REF_INSN (chain->ref)))
	continue;

      if (!DF_REF_REG_MEM_P (chain->ref))
	{
	  if (bitmap_bit_p (insns, uid))
	    continue;

	  if (bitmap_bit_p (candidates, uid))
	    {
	      add_to_queue (uid);
	      continue;
	    }
	}

      if (DF_REF_REG_DEF_P (chain->ref))
	{
	  if (dump_file)
	    fprintf (dump_file, "  r%d def in insn %d isn't convertible\n",
		     DF_REF_REGNO (chain->ref), uid);
	  mark_dual_mode_def (chain->ref);
	}
      else
	{
	  if (dump_file)
	    fprintf (dump_file, "  r%d use in insn %d isn't convertible\n",
		     DF_REF_REGNO (chain->ref), uid);
	  mark_dual_mode_def (ref);
	}
    }
}

/* Add instruction into a chain.  */

void
scalar_chain::add_insn (bitmap candidates, unsigned int insn_uid)
{
  if (bitmap_bit_p (insns, insn_uid))
    return;

  if (dump_file)
    fprintf (dump_file, "  Adding insn %d to chain #%d\n", insn_uid, chain_id);

  bitmap_set_bit (insns, insn_uid);

  rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
  rtx def_set = single_set (insn);
  if (def_set && REG_P (SET_DEST (def_set))
      && !HARD_REGISTER_P (SET_DEST (def_set)))
    bitmap_set_bit (defs, REGNO (SET_DEST (def_set)));

  df_ref ref;
  df_ref def;
  for (ref = DF_INSN_UID_DEFS (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
    if (!HARD_REGISTER_P (DF_REF_REG (ref)))
      for (def = DF_REG_DEF_CHAIN (DF_REF_REGNO (ref));
	   def;
	   def = DF_REF_NEXT_REG (def))
	analyze_register_chain (candidates, def);
  for (ref = DF_INSN_UID_USES (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
    if (!DF_REF_REG_MEM_P (ref))
      analyze_register_chain (candidates, ref);
}

/* Build new chain starting from insn INSN_UID recursively
   adding all dependent uses and definitions.  */

void
scalar_chain::build (bitmap candidates, unsigned insn_uid)
{
  queue = BITMAP_ALLOC (NULL);
  bitmap_set_bit (queue, insn_uid);

  if (dump_file)
    fprintf (dump_file, "Building chain #%d...\n", chain_id);

  while (!bitmap_empty_p (queue))
    {
      insn_uid = bitmap_first_set_bit (queue);
      bitmap_clear_bit (queue, insn_uid);
      bitmap_clear_bit (candidates, insn_uid);
      add_insn (candidates, insn_uid);
    }

  if (dump_file)
    {
      fprintf (dump_file, "Collected chain #%d...\n", chain_id);
      fprintf (dump_file, "  insns: ");
      dump_bitmap (dump_file, insns);
      if (!bitmap_empty_p (defs_conv))
	{
	  bitmap_iterator bi;
	  unsigned id;
	  const char *comma = "";
	  fprintf (dump_file, "  defs to convert: ");
	  EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
	    {
	      fprintf (dump_file, "%sr%d", comma, id);
	      comma = ", ";
	    }
	  fprintf (dump_file, "\n");
	}
    }

  BITMAP_FREE (queue);
}

/* Return a cost of building a vector costant
   instead of using a scalar one.  */

int
dimode_scalar_chain::vector_const_cost (rtx exp)
{
  gcc_assert (CONST_INT_P (exp));

  if (standard_sse_constant_p (exp, V2DImode))
    return COSTS_N_INSNS (1);
  return ix86_cost->sse_load[1];
}

/* Compute a gain for chain conversion.  */

int
dimode_scalar_chain::compute_convert_gain ()
{
  bitmap_iterator bi;
  unsigned insn_uid;
  int gain = 0;
  int cost = 0;

  if (dump_file)
    fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);

  EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
    {
      rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
      rtx def_set = single_set (insn);
      rtx src = SET_SRC (def_set);
      rtx dst = SET_DEST (def_set);

      if (REG_P (src) && REG_P (dst))
	gain += COSTS_N_INSNS (2) - ix86_cost->sse_move;
      else if (REG_P (src) && MEM_P (dst))
	gain += 2 * ix86_cost->int_store[2] - ix86_cost->sse_store[1];
      else if (MEM_P (src) && REG_P (dst))
	gain += 2 * ix86_cost->int_load[2] - ix86_cost->sse_load[1];
      else if (GET_CODE (src) == ASHIFT
	       || GET_CODE (src) == LSHIFTRT)
	{
	  gain += ix86_cost->add;
    	  if (CONST_INT_P (XEXP (src, 0)))
	    gain -= vector_const_cost (XEXP (src, 0));
	  if (CONST_INT_P (XEXP (src, 1))
	      && INTVAL (XEXP (src, 1)) >= 32)
	    gain -= COSTS_N_INSNS (1);
	}
      else if (GET_CODE (src) == PLUS
	       || GET_CODE (src) == MINUS
	       || GET_CODE (src) == IOR
	       || GET_CODE (src) == XOR
	       || GET_CODE (src) == AND)
	{
	  gain += ix86_cost->add;
	  /* Additional gain for andnot for targets without BMI.  */
	  if (GET_CODE (XEXP (src, 0)) == NOT
	      && !TARGET_BMI)
	    gain += 2 * ix86_cost->add;

	  if (CONST_INT_P (XEXP (src, 0)))
	    gain -= vector_const_cost (XEXP (src, 0));
	  if (CONST_INT_P (XEXP (src, 1)))
	    gain -= vector_const_cost (XEXP (src, 1));
	}
      else if (GET_CODE (src) == NEG
	       || GET_CODE (src) == NOT)
	gain += ix86_cost->add - COSTS_N_INSNS (1);
      else if (GET_CODE (src) == COMPARE)
	{
	  /* Assume comparison cost is the same.  */
	}
      else if (CONST_INT_P (src))
	{
	  if (REG_P (dst))
	    gain += COSTS_N_INSNS (2);
	  else if (MEM_P (dst))
	    gain += 2 * ix86_cost->int_store[2] - ix86_cost->sse_store[1];
	  gain -= vector_const_cost (src);
	}
      else
	gcc_unreachable ();
    }

  if (dump_file)
    fprintf (dump_file, "  Instruction conversion gain: %d\n", gain);

  EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, insn_uid, bi)
    cost += DF_REG_DEF_COUNT (insn_uid) * ix86_cost->mmxsse_to_integer;

  if (dump_file)
    fprintf (dump_file, "  Registers conversion cost: %d\n", cost);

  gain -= cost;

  if (dump_file)
    fprintf (dump_file, "  Total gain: %d\n", gain);

  return gain;
}

/* Replace REG in X with a V2DI subreg of NEW_REG.  */

rtx
dimode_scalar_chain::replace_with_subreg (rtx x, rtx reg, rtx new_reg)
{
  if (x == reg)
    return gen_rtx_SUBREG (V2DImode, new_reg, 0);

  const char *fmt = GET_RTX_FORMAT (GET_CODE (x));
  int i, j;
  for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
    {
      if (fmt[i] == 'e')
	XEXP (x, i) = replace_with_subreg (XEXP (x, i), reg, new_reg);
      else if (fmt[i] == 'E')
	for (j = XVECLEN (x, i) - 1; j >= 0; j--)
	  XVECEXP (x, i, j) = replace_with_subreg (XVECEXP (x, i, j),
						   reg, new_reg);
    }

  return x;
}

/* Replace REG in INSN with a V2DI subreg of NEW_REG.  */

void
dimode_scalar_chain::replace_with_subreg_in_insn (rtx_insn *insn,
						  rtx reg, rtx new_reg)
{
  replace_with_subreg (single_set (insn), reg, new_reg);
}

/* Insert generated conversion instruction sequence INSNS
   after instruction AFTER.  New BB may be required in case
   instruction has EH region attached.  */

void
scalar_chain::emit_conversion_insns (rtx insns, rtx_insn *after)
{
  if (!control_flow_insn_p (after))
    {
      emit_insn_after (insns, after);
      return;
    }

  basic_block bb = BLOCK_FOR_INSN (after);
  edge e = find_fallthru_edge (bb->succs);
  gcc_assert (e);

  basic_block new_bb = split_edge (e);
  emit_insn_after (insns, BB_HEAD (new_bb));
}

/* Make vector copies for all register REGNO definitions
   and replace its uses in a chain.  */

void
dimode_scalar_chain::make_vector_copies (unsigned regno)
{
  rtx reg = regno_reg_rtx[regno];
  rtx vreg = gen_reg_rtx (DImode);
  df_ref ref;

  for (ref = DF_REG_DEF_CHAIN (regno); ref; ref = DF_REF_NEXT_REG (ref))
    if (!bitmap_bit_p (insns, DF_REF_INSN_UID (ref)))
      {
	rtx_insn *insn = DF_REF_INSN (ref);

	start_sequence ();
	if (TARGET_SSE4_1)
	  {
	    emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
					CONST0_RTX (V4SImode),
					gen_rtx_SUBREG (SImode, reg, 0)));
	    emit_insn (gen_sse4_1_pinsrd (gen_rtx_SUBREG (V4SImode, vreg, 0),
					  gen_rtx_SUBREG (V4SImode, vreg, 0),
					  gen_rtx_SUBREG (SImode, reg, 4),
					  GEN_INT (2)));
	  }
	else if (TARGET_INTER_UNIT_MOVES_TO_VEC)
	  {
	    rtx tmp = gen_reg_rtx (DImode);
	    emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
					CONST0_RTX (V4SImode),
					gen_rtx_SUBREG (SImode, reg, 0)));
	    emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, tmp, 0),
					CONST0_RTX (V4SImode),
					gen_rtx_SUBREG (SImode, reg, 4)));
	    emit_insn (gen_vec_interleave_lowv4si
		       (gen_rtx_SUBREG (V4SImode, vreg, 0),
			gen_rtx_SUBREG (V4SImode, vreg, 0),
			gen_rtx_SUBREG (V4SImode, tmp, 0)));
	  }
	else
	  {
	    rtx tmp = assign_386_stack_local (DImode, SLOT_STV_TEMP);
	    emit_move_insn (adjust_address (tmp, SImode, 0),
			    gen_rtx_SUBREG (SImode, reg, 0));
	    emit_move_insn (adjust_address (tmp, SImode, 4),
			    gen_rtx_SUBREG (SImode, reg, 4));
	    emit_move_insn (vreg, tmp);
	  }
	rtx_insn *seq = get_insns ();
	end_sequence ();
	emit_conversion_insns (seq, insn);

	if (dump_file)
	  fprintf (dump_file,
		   "  Copied r%d to a vector register r%d for insn %d\n",
		   regno, REGNO (vreg), DF_REF_INSN_UID (ref));
      }

  for (ref = DF_REG_USE_CHAIN (regno); ref; ref = DF_REF_NEXT_REG (ref))
    if (bitmap_bit_p (insns, DF_REF_INSN_UID (ref)))
      {
	replace_with_subreg_in_insn (DF_REF_INSN (ref), reg, vreg);

	if (dump_file)
	  fprintf (dump_file, "  Replaced r%d with r%d in insn %d\n",
		   regno, REGNO (vreg), DF_REF_INSN_UID (ref));
      }
}

/* Convert all definitions of register REGNO
   and fix its uses.  Scalar copies may be created
   in case register is used in not convertible insn.  */

void
dimode_scalar_chain::convert_reg (unsigned regno)
{
  bool scalar_copy = bitmap_bit_p (defs_conv, regno);
  rtx reg = regno_reg_rtx[regno];
  rtx scopy = NULL_RTX;
  df_ref ref;
  bitmap conv;

  conv = BITMAP_ALLOC (NULL);
  bitmap_copy (conv, insns);

  if (scalar_copy)
    scopy = gen_reg_rtx (DImode);

  for (ref = DF_REG_DEF_CHAIN (regno); ref; ref = DF_REF_NEXT_REG (ref))
    {
      rtx_insn *insn = DF_REF_INSN (ref);
      rtx def_set = single_set (insn);
      rtx src = SET_SRC (def_set);
      rtx reg = DF_REF_REG (ref);

      if (!MEM_P (src))
	{
	  replace_with_subreg_in_insn (insn, reg, reg);
	  bitmap_clear_bit (conv, INSN_UID (insn));
	}

      if (scalar_copy)
	{
	  start_sequence ();
	  if (TARGET_SSE4_1)
	    {
	      rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
	      emit_insn
		(gen_rtx_SET
		 (gen_rtx_SUBREG (SImode, scopy, 0),
		  gen_rtx_VEC_SELECT (SImode,
				      gen_rtx_SUBREG (V4SImode, reg, 0), tmp)));

	      tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const1_rtx));
	      emit_insn
		(gen_rtx_SET
		 (gen_rtx_SUBREG (SImode, scopy, 4),
		  gen_rtx_VEC_SELECT (SImode,
				      gen_rtx_SUBREG (V4SImode, reg, 0), tmp)));
	    }
	  else if (TARGET_INTER_UNIT_MOVES_FROM_VEC)
	    {
	      rtx vcopy = gen_reg_rtx (V2DImode);
	      emit_move_insn (vcopy, gen_rtx_SUBREG (V2DImode, reg, 0));
	      emit_move_insn (gen_rtx_SUBREG (SImode, scopy, 0),
			      gen_rtx_SUBREG (SImode, vcopy, 0));
	      emit_move_insn (vcopy,
			      gen_rtx_LSHIFTRT (V2DImode, vcopy, GEN_INT (32)));
	      emit_move_insn (gen_rtx_SUBREG (SImode, scopy, 4),
			      gen_rtx_SUBREG (SImode, vcopy, 0));
	    }
	  else
	    {
	      rtx tmp = assign_386_stack_local (DImode, SLOT_STV_TEMP);
	      emit_move_insn (tmp, reg);
	      emit_move_insn (gen_rtx_SUBREG (SImode, scopy, 0),
			      adjust_address (tmp, SImode, 0));
	      emit_move_insn (gen_rtx_SUBREG (SImode, scopy, 4),
			      adjust_address (tmp, SImode, 4));
	    }
	  rtx_insn *seq = get_insns ();
	  end_sequence ();
	  emit_conversion_insns (seq, insn);

	  if (dump_file)
	    fprintf (dump_file,
		     "  Copied r%d to a scalar register r%d for insn %d\n",
		     regno, REGNO (scopy), INSN_UID (insn));
	}
    }

  for (ref = DF_REG_USE_CHAIN (regno); ref; ref = DF_REF_NEXT_REG (ref))
    if (bitmap_bit_p (insns, DF_REF_INSN_UID (ref)))
      {
	if (bitmap_bit_p (conv, DF_REF_INSN_UID (ref)))
	  {
	    rtx def_set = single_set (DF_REF_INSN (ref));
	    if (!MEM_P (SET_DEST (def_set))
		|| !REG_P (SET_SRC (def_set)))
	      replace_with_subreg_in_insn (DF_REF_INSN (ref), reg, reg);
	    bitmap_clear_bit (conv, DF_REF_INSN_UID (ref));
	  }
      }
    /* Skip debug insns and uninitialized uses.  */
    else if (DF_REF_CHAIN (ref)
	     && NONDEBUG_INSN_P (DF_REF_INSN (ref)))
      {
	gcc_assert (scopy);
	replace_rtx (DF_REF_INSN (ref), reg, scopy);
	df_insn_rescan (DF_REF_INSN (ref));
      }

  BITMAP_FREE (conv);
}

/* Convert operand OP in INSN.  We should handle
   memory operands and uninitialized registers.
   All other register uses are converted during
   registers conversion.  */

void
dimode_scalar_chain::convert_op (rtx *op, rtx_insn *insn)
{
  *op = copy_rtx_if_shared (*op);

  if (GET_CODE (*op) == NOT)
    {
      convert_op (&XEXP (*op, 0), insn);
      PUT_MODE (*op, V2DImode);
    }
  else if (MEM_P (*op))
    {
      rtx tmp = gen_reg_rtx (DImode);

      emit_insn_before (gen_move_insn (tmp, *op), insn);
      *op = gen_rtx_SUBREG (V2DImode, tmp, 0);

      if (dump_file)
	fprintf (dump_file, "  Preloading operand for insn %d into r%d\n",
		 INSN_UID (insn), REGNO (tmp));
    }
  else if (REG_P (*op))
    {
      /* We may have not converted register usage in case
	 this register has no definition.  Otherwise it
	 should be converted in convert_reg.  */
      df_ref ref;
      FOR_EACH_INSN_USE (ref, insn)
	if (DF_REF_REGNO (ref) == REGNO (*op))
	  {
	    gcc_assert (!DF_REF_CHAIN (ref));
	    break;
	  }
      *op = gen_rtx_SUBREG (V2DImode, *op, 0);
    }
  else if (CONST_INT_P (*op))
    {
      rtx vec_cst;
      rtx tmp = gen_rtx_SUBREG (V2DImode, gen_reg_rtx (DImode), 0);

      /* Prefer all ones vector in case of -1.  */
      if (constm1_operand (*op, GET_MODE (*op)))
	vec_cst = CONSTM1_RTX (V2DImode);
      else
	vec_cst = gen_rtx_CONST_VECTOR (V2DImode,
					gen_rtvec (2, *op, const0_rtx));

      if (!standard_sse_constant_p (vec_cst, V2DImode))
	{
	  start_sequence ();
	  vec_cst = validize_mem (force_const_mem (V2DImode, vec_cst));
	  rtx_insn *seq = get_insns ();
	  end_sequence ();
	  emit_insn_before (seq, insn);
	}

      emit_insn_before (gen_move_insn (copy_rtx (tmp), vec_cst), insn);
      *op = tmp;
    }
  else
    {
      gcc_assert (SUBREG_P (*op));
      gcc_assert (GET_MODE (*op) == V2DImode);
    }
}

/* Convert INSN to vector mode.  */

void
dimode_scalar_chain::convert_insn (rtx_insn *insn)
{
  rtx def_set = single_set (insn);
  rtx src = SET_SRC (def_set);
  rtx dst = SET_DEST (def_set);
  rtx subreg;

  if (MEM_P (dst) && !REG_P (src))
    {
      /* There are no scalar integer instructions and therefore
	 temporary register usage is required.  */
      rtx tmp = gen_reg_rtx (DImode);
      emit_conversion_insns (gen_move_insn (dst, tmp), insn);
      dst = gen_rtx_SUBREG (V2DImode, tmp, 0);
    }

  switch (GET_CODE (src))
    {
    case ASHIFT:
    case LSHIFTRT:
      convert_op (&XEXP (src, 0), insn);
      PUT_MODE (src, V2DImode);
      break;

    case PLUS:
    case MINUS:
    case IOR:
    case XOR:
    case AND:
      convert_op (&XEXP (src, 0), insn);
      convert_op (&XEXP (src, 1), insn);
      PUT_MODE (src, V2DImode);
      break;

    case NEG:
      src = XEXP (src, 0);
      convert_op (&src, insn);
      subreg = gen_reg_rtx (V2DImode);
      emit_insn_before (gen_move_insn (subreg, CONST0_RTX (V2DImode)), insn);
      src = gen_rtx_MINUS (V2DImode, subreg, src);
      break;

    case NOT:
      src = XEXP (src, 0);
      convert_op (&src, insn);
      subreg = gen_reg_rtx (V2DImode);
      emit_insn_before (gen_move_insn (subreg, CONSTM1_RTX (V2DImode)), insn);
      src = gen_rtx_XOR (V2DImode, src, subreg);
      break;

    case MEM:
      if (!REG_P (dst))
	convert_op (&src, insn);
      break;

    case REG:
      if (!MEM_P (dst))
	convert_op (&src, insn);
      break;

    case SUBREG:
      gcc_assert (GET_MODE (src) == V2DImode);
      break;

    case COMPARE:
      src = SUBREG_REG (XEXP (XEXP (src, 0), 0));

      gcc_assert ((REG_P (src) && GET_MODE (src) == DImode)
		  || (SUBREG_P (src) && GET_MODE (src) == V2DImode));

      if (REG_P (src))
	subreg = gen_rtx_SUBREG (V2DImode, src, 0);
      else
	subreg = copy_rtx_if_shared (src);
      emit_insn_before (gen_vec_interleave_lowv2di (copy_rtx_if_shared (subreg),
						    copy_rtx_if_shared (subreg),
						    copy_rtx_if_shared (subreg)),
			insn);
      dst = gen_rtx_REG (CCmode, FLAGS_REG);
      src = gen_rtx_UNSPEC (CCmode, gen_rtvec (2, copy_rtx_if_shared (src),
					       copy_rtx_if_shared (src)),
			    UNSPEC_PTEST);
      break;

    case CONST_INT:
      convert_op (&src, insn);
      break;

    default:
      gcc_unreachable ();
    }

  SET_SRC (def_set) = src;
  SET_DEST (def_set) = dst;

  /* Drop possible dead definitions.  */
  PATTERN (insn) = def_set;

  INSN_CODE (insn) = -1;
  recog_memoized (insn);
  df_insn_rescan (insn);
}

/* Fix uses of converted REG in debug insns.  */

void
timode_scalar_chain::fix_debug_reg_uses (rtx reg)
{
  if (!flag_var_tracking)
    return;

  df_ref ref, next;
  for (ref = DF_REG_USE_CHAIN (REGNO (reg)); ref; ref = next)
    {
      rtx_insn *insn = DF_REF_INSN (ref);
      /* Make sure the next ref is for a different instruction,
         so that we're not affected by the rescan.  */
      next = DF_REF_NEXT_REG (ref);
      while (next && DF_REF_INSN (next) == insn)
	next = DF_REF_NEXT_REG (next);

      if (DEBUG_INSN_P (insn))
	{
	  /* It may be a debug insn with a TImode variable in
	     register.  */
	  bool changed = false;
	  for (; ref != next; ref = DF_REF_NEXT_REG (ref))
	    {
	      rtx *loc = DF_REF_LOC (ref);
	      if (REG_P (*loc) && GET_MODE (*loc) == V1TImode)
		{
		  *loc = gen_rtx_SUBREG (TImode, *loc, 0);
		  changed = true;
		}
	    }
	  if (changed)
	    df_insn_rescan (insn);
	}
    }
}

/* Convert INSN from TImode to V1T1mode.  */

void
timode_scalar_chain::convert_insn (rtx_insn *insn)
{
  rtx def_set = single_set (insn);
  rtx src = SET_SRC (def_set);
  rtx dst = SET_DEST (def_set);

  switch (GET_CODE (dst))
    {
    case REG:
      {
	rtx tmp = find_reg_equal_equiv_note (insn);
	if (tmp)
	  PUT_MODE (XEXP (tmp, 0), V1TImode);
	PUT_MODE (dst, V1TImode);
	fix_debug_reg_uses (dst);
      }
      break;
    case MEM:
      PUT_MODE (dst, V1TImode);
      break;

    default:
      gcc_unreachable ();
    }

  switch (GET_CODE (src))
    {
    case REG:
      PUT_MODE (src, V1TImode);
      /* Call fix_debug_reg_uses only if SRC is never defined.  */
      if (!DF_REG_DEF_CHAIN (REGNO (src)))
	fix_debug_reg_uses (src);
      break;

    case MEM:
      PUT_MODE (src, V1TImode);
      break;

    case CONST_WIDE_INT:
      if (NONDEBUG_INSN_P (insn))
	{
	  /* Since there are no instructions to store 128-bit constant,
	     temporary register usage is required.  */
	  rtx tmp = gen_reg_rtx (V1TImode);
	  start_sequence ();
	  src = gen_rtx_CONST_VECTOR (V1TImode, gen_rtvec (1, src));
	  src = validize_mem (force_const_mem (V1TImode, src));
	  rtx_insn *seq = get_insns ();
	  end_sequence ();
	  if (seq)
	    emit_insn_before (seq, insn);
	  emit_conversion_insns (gen_rtx_SET (dst, tmp), insn);
	  dst = tmp;
	}
      break;

    case CONST_INT:
      switch (standard_sse_constant_p (src, TImode))
	{
	case 1:
	  src = CONST0_RTX (GET_MODE (dst));
	  break;
	case 2:
	  src = CONSTM1_RTX (GET_MODE (dst));
	  break;
	default:
	  gcc_unreachable ();
	}
      if (NONDEBUG_INSN_P (insn))
	{
	  rtx tmp = gen_reg_rtx (V1TImode);
	  /* Since there are no instructions to store standard SSE
	     constant, temporary register usage is required.  */
	  emit_conversion_insns (gen_rtx_SET (dst, tmp), insn);
	  dst = tmp;
	}
      break;

    default:
      gcc_unreachable ();
    }

  SET_SRC (def_set) = src;
  SET_DEST (def_set) = dst;

  /* Drop possible dead definitions.  */
  PATTERN (insn) = def_set;

  INSN_CODE (insn) = -1;
  recog_memoized (insn);
  df_insn_rescan (insn);
}

void
dimode_scalar_chain::convert_registers ()
{
  bitmap_iterator bi;
  unsigned id;

  EXECUTE_IF_SET_IN_BITMAP (defs, 0, id, bi)
    convert_reg (id);

  EXECUTE_IF_AND_COMPL_IN_BITMAP (defs_conv, defs, 0, id, bi)
    make_vector_copies (id);
}

/* Convert whole chain creating required register
   conversions and copies.  */

int
scalar_chain::convert ()
{
  bitmap_iterator bi;
  unsigned id;
  int converted_insns = 0;

  if (!dbg_cnt (stv_conversion))
    return 0;

  if (dump_file)
    fprintf (dump_file, "Converting chain #%d...\n", chain_id);

  convert_registers ();

  EXECUTE_IF_SET_IN_BITMAP (insns, 0, id, bi)
    {
      convert_insn (DF_INSN_UID_GET (id)->insn);
      converted_insns++;
    }

  return converted_insns;
}

/* Main STV pass function.  Find and convert scalar
   instructions into vector mode when profitable.  */

static unsigned int
convert_scalars_to_vector ()
{
  basic_block bb;
  bitmap candidates;
  int converted_insns = 0;

  bitmap_obstack_initialize (NULL);
  candidates = BITMAP_ALLOC (NULL);

  calculate_dominance_info (CDI_DOMINATORS);
  df_set_flags (DF_DEFER_INSN_RESCAN);
  df_chain_add_problem (DF_DU_CHAIN | DF_UD_CHAIN);
  df_md_add_problem ();
  df_analyze ();

  /* Find all instructions we want to convert into vector mode.  */
  if (dump_file)
    fprintf (dump_file, "Searching for mode conversion candidates...\n");

  FOR_EACH_BB_FN (bb, cfun)
    {
      rtx_insn *insn;
      FOR_BB_INSNS (bb, insn)
	if (scalar_to_vector_candidate_p (insn))
	  {
	    if (dump_file)
	      fprintf (dump_file, "  insn %d is marked as a candidate\n",
		       INSN_UID (insn));

	    bitmap_set_bit (candidates, INSN_UID (insn));
	  }
    }

  remove_non_convertible_regs (candidates);

  if (bitmap_empty_p (candidates))
    if (dump_file)
      fprintf (dump_file, "There are no candidates for optimization.\n");

  while (!bitmap_empty_p (candidates))
    {
      unsigned uid = bitmap_first_set_bit (candidates);
      scalar_chain *chain;

      if (TARGET_64BIT)
	chain = new timode_scalar_chain;
      else
	chain = new dimode_scalar_chain;

      /* Find instructions chain we want to convert to vector mode.
	 Check all uses and definitions to estimate all required
	 conversions.  */
      chain->build (candidates, uid);

      if (chain->compute_convert_gain () > 0)
	converted_insns += chain->convert ();
      else
	if (dump_file)
	  fprintf (dump_file, "Chain #%d conversion is not profitable\n",
		   chain->chain_id);

      delete chain;
    }

  if (dump_file)
    fprintf (dump_file, "Total insns converted: %d\n", converted_insns);

  BITMAP_FREE (candidates);
  bitmap_obstack_release (NULL);
  df_process_deferred_rescans ();

  /* Conversion means we may have 128bit register spills/fills
     which require aligned stack.  */
  if (converted_insns)
    {
      if (crtl->stack_alignment_needed < 128)
	crtl->stack_alignment_needed = 128;
      if (crtl->stack_alignment_estimated < 128)
	crtl->stack_alignment_estimated = 128;
      /* Fix up DECL_RTL/DECL_INCOMING_RTL of arguments.  */
      if (TARGET_64BIT)
	for (tree parm = DECL_ARGUMENTS (current_function_decl);
	     parm; parm = DECL_CHAIN (parm))
	  {
	    if (TYPE_MODE (TREE_TYPE (parm)) != TImode)
	      continue;
	    if (DECL_RTL_SET_P (parm)
		&& GET_MODE (DECL_RTL (parm)) == V1TImode)
	      {
		rtx r = DECL_RTL (parm);
		if (REG_P (r))
		  SET_DECL_RTL (parm, gen_rtx_SUBREG (TImode, r, 0));
	      }
	    if (DECL_INCOMING_RTL (parm)
		&& GET_MODE (DECL_INCOMING_RTL (parm)) == V1TImode)
	      {
		rtx r = DECL_INCOMING_RTL (parm);
		if (REG_P (r))
		  DECL_INCOMING_RTL (parm) = gen_rtx_SUBREG (TImode, r, 0);
	      }
	  }
    }

  return 0;
}

namespace {

const pass_data pass_data_insert_vzeroupper =
{
  RTL_PASS, /* type */
  "vzeroupper", /* name */
  OPTGROUP_NONE, /* optinfo_flags */
  TV_MACH_DEP, /* tv_id */
  0, /* properties_required */
  0, /* properties_provided */
  0, /* properties_destroyed */
  0, /* todo_flags_start */
  TODO_df_finish, /* todo_flags_finish */
};

class pass_insert_vzeroupper : public rtl_opt_pass
{
public:
  pass_insert_vzeroupper(gcc::context *ctxt)
    : rtl_opt_pass(pass_data_insert_vzeroupper, ctxt)
  {}

  /* opt_pass methods: */
  virtual bool gate (function *)
    {
      return TARGET_AVX && !TARGET_AVX512F
	     && TARGET_VZEROUPPER && flag_expensive_optimizations
	     && !optimize_size;
    }

  virtual unsigned int execute (function *)
    {
      return rest_of_handle_insert_vzeroupper ();
    }

}; // class pass_insert_vzeroupper

const pass_data pass_data_stv =
{
  RTL_PASS, /* type */
  "stv", /* name */
  OPTGROUP_NONE, /* optinfo_flags */
  TV_MACH_DEP, /* tv_id */
  0, /* properties_required */
  0, /* properties_provided */
  0, /* properties_destroyed */
  0, /* todo_flags_start */
  TODO_df_finish, /* todo_flags_finish */
};

class pass_stv : public rtl_opt_pass
{
public:
  pass_stv (gcc::context *ctxt)
    : rtl_opt_pass (pass_data_stv, ctxt),
      timode_p (false)
  {}

  /* opt_pass methods: */
  virtual bool gate (function *)
    {
      return (timode_p == !!TARGET_64BIT
	      && TARGET_STV && TARGET_SSE2 && optimize > 1);
    }

  virtual unsigned int execute (function *)
    {
      return convert_scalars_to_vector ();
    }

  opt_pass *clone ()
    {
      return new pass_stv (m_ctxt);
    }

  void set_pass_param (unsigned int n, bool param)
    {
      gcc_assert (n == 0);
      timode_p = param;
    }

private:
  bool timode_p;
}; // class pass_stv

} // anon namespace

rtl_opt_pass *
make_pass_insert_vzeroupper (gcc::context *ctxt)
{
  return new pass_insert_vzeroupper (ctxt);
}

rtl_opt_pass *
make_pass_stv (gcc::context *ctxt)
{
  return new pass_stv (ctxt);
}

/* Return true if a red-zone is in use.  */

bool
ix86_using_red_zone (void)
{
  return TARGET_RED_ZONE && !TARGET_64BIT_MS_ABI;
}

/* Return a string that documents the current -m options.  The caller is
   responsible for freeing the string.  */

static char *
ix86_target_string (HOST_WIDE_INT isa, HOST_WIDE_INT isa2,
		    int flags, int flags2,
		    const char *arch, const char *tune,
		    enum fpmath_unit fpmath, bool add_nl_p)
{
  struct ix86_target_opts
  {
    const char *option;		/* option string */
    HOST_WIDE_INT mask;		/* isa mask options */
  };

  /* This table is ordered so that options like -msse4.2 that imply other
     ISAs come first.  Target string will be displayed in the same order.  */
  static struct ix86_target_opts isa2_opts[] =
  {
    { "-mrdpid",	OPTION_MASK_ISA_RDPID },
    { "-msgx",		OPTION_MASK_ISA_SGX },
    { "-mavx5124vnniw", OPTION_MASK_ISA_AVX5124VNNIW },
    { "-mavx5124fmaps", OPTION_MASK_ISA_AVX5124FMAPS },
    { "-mavx512vpopcntdq", OPTION_MASK_ISA_AVX512VPOPCNTDQ }
  };
  static struct ix86_target_opts isa_opts[] =
  {
    { "-mavx512vbmi",	OPTION_MASK_ISA_AVX512VBMI },
    { "-mavx512ifma",	OPTION_MASK_ISA_AVX512IFMA },
    { "-mavx512vl",	OPTION_MASK_ISA_AVX512VL },
    { "-mavx512bw",	OPTION_MASK_ISA_AVX512BW },
    { "-mavx512dq",	OPTION_MASK_ISA_AVX512DQ },
    { "-mavx512er",	OPTION_MASK_ISA_AVX512ER },
    { "-mavx512pf",	OPTION_MASK_ISA_AVX512PF },
    { "-mavx512cd",	OPTION_MASK_ISA_AVX512CD },
    { "-mavx512f",	OPTION_MASK_ISA_AVX512F },
    { "-mavx2",		OPTION_MASK_ISA_AVX2 },
    { "-mfma",		OPTION_MASK_ISA_FMA },
    { "-mxop",		OPTION_MASK_ISA_XOP },
    { "-mfma4",		OPTION_MASK_ISA_FMA4 },
    { "-mf16c",		OPTION_MASK_ISA_F16C },
    { "-mavx",		OPTION_MASK_ISA_AVX },
/*  { "-msse4"		OPTION_MASK_ISA_SSE4 }, */
    { "-msse4.2",	OPTION_MASK_ISA_SSE4_2 },
    { "-msse4.1",	OPTION_MASK_ISA_SSE4_1 },
    { "-msse4a",	OPTION_MASK_ISA_SSE4A },
    { "-mssse3",	OPTION_MASK_ISA_SSSE3 },
    { "-msse3",		OPTION_MASK_ISA_SSE3 },
    { "-maes",		OPTION_MASK_ISA_AES },
    { "-msha",		OPTION_MASK_ISA_SHA },
    { "-mpclmul",	OPTION_MASK_ISA_PCLMUL },
    { "-msse2",		OPTION_MASK_ISA_SSE2 },
    { "-msse",		OPTION_MASK_ISA_SSE },
    { "-m3dnowa",	OPTION_MASK_ISA_3DNOW_A },
    { "-m3dnow",	OPTION_MASK_ISA_3DNOW },
    { "-mmmx",		OPTION_MASK_ISA_MMX },
    { "-mrtm",		OPTION_MASK_ISA_RTM },
    { "-mprfchw",	OPTION_MASK_ISA_PRFCHW },
    { "-mrdseed",	OPTION_MASK_ISA_RDSEED },
    { "-madx",		OPTION_MASK_ISA_ADX },
    { "-mprefetchwt1",	OPTION_MASK_ISA_PREFETCHWT1 },
    { "-mclflushopt",	OPTION_MASK_ISA_CLFLUSHOPT },
    { "-mxsaves",	OPTION_MASK_ISA_XSAVES },
    { "-mxsavec",	OPTION_MASK_ISA_XSAVEC },
    { "-mxsaveopt",	OPTION_MASK_ISA_XSAVEOPT },
    { "-mxsave",	OPTION_MASK_ISA_XSAVE },
    { "-mabm",		OPTION_MASK_ISA_ABM },
    { "-mbmi",		OPTION_MASK_ISA_BMI },
    { "-mbmi2",		OPTION_MASK_ISA_BMI2 },
    { "-mlzcnt",	OPTION_MASK_ISA_LZCNT },
    { "-mtbm",		OPTION_MASK_ISA_TBM },
    { "-mpopcnt",	OPTION_MASK_ISA_POPCNT },
    { "-mcx16",		OPTION_MASK_ISA_CX16 },
    { "-msahf",		OPTION_MASK_ISA_SAHF },
    { "-mmovbe",	OPTION_MASK_ISA_MOVBE },
    { "-mcrc32",	OPTION_MASK_ISA_CRC32 },
    { "-mfsgsbase",	OPTION_MASK_ISA_FSGSBASE },
    { "-mrdrnd",	OPTION_MASK_ISA_RDRND },
    { "-mmwaitx",	OPTION_MASK_ISA_MWAITX },
    { "-mclzero",	OPTION_MASK_ISA_CLZERO },
    { "-mpku",		OPTION_MASK_ISA_PKU },
    { "-mlwp",		OPTION_MASK_ISA_LWP },
    { "-mhle",		OPTION_MASK_ISA_HLE },
    { "-mfxsr",		OPTION_MASK_ISA_FXSR },
    { "-mmpx",		OPTION_MASK_ISA_MPX },
    { "-mclwb",		OPTION_MASK_ISA_CLWB }
  };

  /* Flag options.  */
  static struct ix86_target_opts flag_opts[] =
  {
    { "-m128bit-long-double",		MASK_128BIT_LONG_DOUBLE },
    { "-mlong-double-128",		MASK_LONG_DOUBLE_128 },
    { "-mlong-double-64",		MASK_LONG_DOUBLE_64 },
    { "-m80387",			MASK_80387 },
    { "-maccumulate-outgoing-args",	MASK_ACCUMULATE_OUTGOING_ARGS },
    { "-malign-double",			MASK_ALIGN_DOUBLE },
    { "-mcld",				MASK_CLD },
    { "-mfp-ret-in-387",		MASK_FLOAT_RETURNS },
    { "-mieee-fp",			MASK_IEEE_FP },
    { "-minline-all-stringops",		MASK_INLINE_ALL_STRINGOPS },
    { "-minline-stringops-dynamically",	MASK_INLINE_STRINGOPS_DYNAMICALLY },
    { "-mms-bitfields",			MASK_MS_BITFIELD_LAYOUT },
    { "-mno-align-stringops",		MASK_NO_ALIGN_STRINGOPS },
    { "-mno-fancy-math-387",		MASK_NO_FANCY_MATH_387 },
    { "-mno-push-args",			MASK_NO_PUSH_ARGS },
    { "-mno-red-zone",			MASK_NO_RED_ZONE },
    { "-momit-leaf-frame-pointer",	MASK_OMIT_LEAF_FRAME_POINTER },
    { "-mrecip",			MASK_RECIP },
    { "-mrtd",				MASK_RTD },
    { "-msseregparm",			MASK_SSEREGPARM },
    { "-mstack-arg-probe",		MASK_STACK_PROBE },
    { "-mtls-direct-seg-refs",		MASK_TLS_DIRECT_SEG_REFS },
    { "-mvect8-ret-in-mem",		MASK_VECT8_RETURNS },
    { "-m8bit-idiv",			MASK_USE_8BIT_IDIV },
    { "-mvzeroupper",			MASK_VZEROUPPER },
    { "-mstv",				MASK_STV },
    { "-mavx256-split-unaligned-load",	MASK_AVX256_SPLIT_UNALIGNED_LOAD },
    { "-mavx256-split-unaligned-store",	MASK_AVX256_SPLIT_UNALIGNED_STORE },
    { "-mprefer-avx128",		MASK_PREFER_AVX128 }
  };

  /* Additional flag options.  */
  static struct ix86_target_opts flag2_opts[] =
  {
    { "-mgeneral-regs-only",		OPTION_MASK_GENERAL_REGS_ONLY },
  };

  const char *opts[ARRAY_SIZE (isa_opts) + ARRAY_SIZE (isa2_opts)
		   + ARRAY_SIZE (flag_opts) + ARRAY_SIZE (flag2_opts) + 6][2];

  char isa_other[40];
  char isa2_other[40];
  char flags_other[40];
  char flags2_other[40];
  unsigned num = 0;
  unsigned i, j;
  char *ret;
  char *ptr;
  size_t len;
  size_t line_len;
  size_t sep_len;
  const char *abi;

  memset (opts, '\0', sizeof (opts));

  /* Add -march= option.  */
  if (arch)
    {
      opts[num][0] = "-march=";
      opts[num++][1] = arch;
    }

  /* Add -mtune= option.  */
  if (tune)
    {
      opts[num][0] = "-mtune=";
      opts[num++][1] = tune;
    }

  /* Add -m32/-m64/-mx32.  */
  if ((isa & OPTION_MASK_ISA_64BIT) != 0)
    {
      if ((isa & OPTION_MASK_ABI_64) != 0)
	abi = "-m64";
      else
	abi = "-mx32";
      isa &= ~ (OPTION_MASK_ISA_64BIT
		| OPTION_MASK_ABI_64
		| OPTION_MASK_ABI_X32);
    }
  else
    abi = "-m32";
  opts[num++][0] = abi;

  /* Pick out the options in isa2 options.  */
  for (i = 0; i < ARRAY_SIZE (isa2_opts); i++)
    {
      if ((isa2 & isa2_opts[i].mask) != 0)
	{
	  opts[num++][0] = isa2_opts[i].option;
	  isa2 &= ~ isa2_opts[i].mask;
	}
    }

  if (isa2 && add_nl_p)
    {
      opts[num++][0] = isa2_other;
      sprintf (isa2_other, "(other isa2: %#" HOST_WIDE_INT_PRINT "x)", isa2);
    }

  /* Pick out the options in isa options.  */
  for (i = 0; i < ARRAY_SIZE (isa_opts); i++)
    {
      if ((isa & isa_opts[i].mask) != 0)
	{
	  opts[num++][0] = isa_opts[i].option;
	  isa &= ~ isa_opts[i].mask;
	}
    }

  if (isa && add_nl_p)
    {
      opts[num++][0] = isa_other;
      sprintf (isa_other, "(other isa: %#" HOST_WIDE_INT_PRINT "x)", isa);
    }

  /* Add flag options.  */
  for (i = 0; i < ARRAY_SIZE (flag_opts); i++)
    {
      if ((flags & flag_opts[i].mask) != 0)
	{
	  opts[num++][0] = flag_opts[i].option;
	  flags &= ~ flag_opts[i].mask;
	}
    }

  if (flags && add_nl_p)
    {
      opts[num++][0] = flags_other;
      sprintf (flags_other, "(other flags: %#x)", flags);
    }

    /* Add additional flag options.  */
  for (i = 0; i < ARRAY_SIZE (flag2_opts); i++)
    {
      if ((flags2 & flag2_opts[i].mask) != 0)
	{
	  opts[num++][0] = flag2_opts[i].option;
	  flags2 &= ~ flag2_opts[i].mask;
	}
    }

  if (flags2 && add_nl_p)
    {
      opts[num++][0] = flags2_other;
      sprintf (flags2_other, "(other flags2: %#x)", flags2);
    }

  /* Add -fpmath= option.  */
  if (fpmath)
    {
      opts[num][0] = "-mfpmath=";
      switch ((int) fpmath)
	{
	case FPMATH_387:
	  opts[num++][1] = "387";
	  break;

	case FPMATH_SSE:
	  opts[num++][1] = "sse";
	  break;

	case FPMATH_387 | FPMATH_SSE:
	  opts[num++][1] = "sse+387";
	  break;

	default:
	  gcc_unreachable ();
	}
    }

  /* Any options?  */
  if (num == 0)
    return NULL;

  gcc_assert (num < ARRAY_SIZE (opts));

  /* Size the string.  */
  len = 0;
  sep_len = (add_nl_p) ? 3 : 1;
  for (i = 0; i < num; i++)
    {
      len += sep_len;
      for (j = 0; j < 2; j++)
	if (opts[i][j])
	  len += strlen (opts[i][j]);
    }

  /* Build the string.  */
  ret = ptr = (char *) xmalloc (len);
  line_len = 0;

  for (i = 0; i < num; i++)
    {
      size_t len2[2];

      for (j = 0; j < 2; j++)
	len2[j] = (opts[i][j]) ? strlen (opts[i][j]) : 0;

      if (i != 0)
	{
	  *ptr++ = ' ';
	  line_len++;

	  if (add_nl_p && line_len + len2[0] + len2[1] > 70)
	    {
	      *ptr++ = '\\';
	      *ptr++ = '\n';
	      line_len = 0;
	    }
	}

      for (j = 0; j < 2; j++)
	if (opts[i][j])
	  {
	    memcpy (ptr, opts[i][j], len2[j]);
	    ptr += len2[j];
	    line_len += len2[j];
	  }
    }

  *ptr = '\0';
  gcc_assert (ret + len >= ptr);

  return ret;
}

/* Return true, if profiling code should be emitted before
   prologue. Otherwise it returns false.
   Note: For x86 with "hotfix" it is sorried.  */
static bool
ix86_profile_before_prologue (void)
{
  return flag_fentry != 0;
}

/* Function that is callable from the debugger to print the current
   options.  */
void ATTRIBUTE_UNUSED
ix86_debug_options (void)
{
  char *opts = ix86_target_string (ix86_isa_flags, ix86_isa_flags2,
				   target_flags, ix86_target_flags,
				   ix86_arch_string,ix86_tune_string,
				   ix86_fpmath, true);

  if (opts)
    {
      fprintf (stderr, "%s\n\n", opts);
      free (opts);
    }
  else
    fputs ("<no options>\n\n", stderr);

  return;
}

/* Return true if T is one of the bytes we should avoid with
   -fmitigate-rop.  */

static bool
ix86_rop_should_change_byte_p (int t)
{
  return t == 0xc2 || t == 0xc3 || t == 0xca || t == 0xcb;
}

static const char *stringop_alg_names[] = {
#define DEF_ENUM
#define DEF_ALG(alg, name) #name,
#include "stringop.def"
#undef DEF_ENUM
#undef DEF_ALG
};

/* Parse parameter string passed to -mmemcpy-strategy= or -mmemset-strategy=.
   The string is of the following form (or comma separated list of it):

     strategy_alg:max_size:[align|noalign]

   where the full size range for the strategy is either [0, max_size] or
   [min_size, max_size], in which min_size is the max_size + 1 of the
   preceding range.  The last size range must have max_size == -1.

   Examples:

    1.
       -mmemcpy-strategy=libcall:-1:noalign

      this is equivalent to (for known size memcpy) -mstringop-strategy=libcall


   2.
      -mmemset-strategy=rep_8byte:16:noalign,vector_loop:2048:align,libcall:-1:noalign

      This is to tell the compiler to use the following strategy for memset
      1) when the expected size is between [1, 16], use rep_8byte strategy;
      2) when the size is between [17, 2048], use vector_loop;
      3) when the size is > 2048, use libcall.  */

struct stringop_size_range
{
  int max;
  stringop_alg alg;
  bool noalign;
};

static void
ix86_parse_stringop_strategy_string (char *strategy_str, bool is_memset)
{
  const struct stringop_algs *default_algs;
  stringop_size_range input_ranges[MAX_STRINGOP_ALGS];
  char *curr_range_str, *next_range_str;
  const char *opt = is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=";
  int i = 0, n = 0;

  if (is_memset)
    default_algs = &ix86_cost->memset[TARGET_64BIT != 0];
  else
    default_algs = &ix86_cost->memcpy[TARGET_64BIT != 0];

  curr_range_str = strategy_str;

  do
    {
      int maxs;
      char alg_name[128];
      char align[16];
      next_range_str = strchr (curr_range_str, ',');
      if (next_range_str)
        *next_range_str++ = '\0';

      if (3 != sscanf (curr_range_str, "%20[^:]:%d:%10s",
                       alg_name, &maxs, align))
        {
	  error ("wrong argument %qs to option %qs", curr_range_str, opt);
          return;
        }

      if (n > 0 && (maxs < (input_ranges[n - 1].max + 1) && maxs != -1))
        {
	  error ("size ranges of option %qs should be increasing", opt);
          return;
        }

      for (i = 0; i < last_alg; i++)
	if (!strcmp (alg_name, stringop_alg_names[i]))
	  break;

      if (i == last_alg)
        {
	  error ("wrong strategy name %qs specified for option %qs",
		 alg_name, opt);

	  auto_vec <const char *> candidates;
	  for (i = 0; i < last_alg; i++)
	    if ((stringop_alg) i != rep_prefix_8_byte || TARGET_64BIT)
	      candidates.safe_push (stringop_alg_names[i]);

	  char *s;
	  const char *hint
	    = candidates_list_and_hint (alg_name, s, candidates);
	  if (hint)
	    inform (input_location,
		    "valid arguments to %qs are: %s; did you mean %qs?",
		    opt, s, hint);
	  else
	    inform (input_location, "valid arguments to %qs are: %s",
		    opt, s);
	  XDELETEVEC (s);
          return;
        }

      if ((stringop_alg) i == rep_prefix_8_byte
	  && !TARGET_64BIT)
	{
	  /* rep; movq isn't available in 32-bit code.  */
	  error ("strategy name %qs specified for option %qs "
		 "not supported for 32-bit code", alg_name, opt);
	  return;
	}

      input_ranges[n].max = maxs;
      input_ranges[n].alg = (stringop_alg) i;
      if (!strcmp (align, "align"))
        input_ranges[n].noalign = false;
      else if (!strcmp (align, "noalign"))
        input_ranges[n].noalign = true;
      else
        {
	  error ("unknown alignment %qs specified for option %qs", align, opt);
          return;
        }
      n++;
      curr_range_str = next_range_str;
    }
  while (curr_range_str);

  if (input_ranges[n - 1].max != -1)
    {
      error ("the max value for the last size range should be -1"
             " for option %qs", opt);
      return;
    }

  if (n > MAX_STRINGOP_ALGS)
    {
      error ("too many size ranges specified in option %qs", opt);
      return;
    }

  /* Now override the default algs array.  */
  for (i = 0; i < n; i++)
    {
      *const_cast<int *>(&default_algs->size[i].max) = input_ranges[i].max;
      *const_cast<stringop_alg *>(&default_algs->size[i].alg)
          = input_ranges[i].alg;
      *const_cast<int *>(&default_algs->size[i].noalign)
          = input_ranges[i].noalign;
    }
}


/* parse -mtune-ctrl= option. When DUMP is true,
   print the features that are explicitly set.  */

static void
parse_mtune_ctrl_str (bool dump)
{
  if (!ix86_tune_ctrl_string)
    return;

  char *next_feature_string = NULL;
  char *curr_feature_string = xstrdup (ix86_tune_ctrl_string);
  char *orig = curr_feature_string;
  int i;
  do
    {
      bool clear = false;

      next_feature_string = strchr (curr_feature_string, ',');
      if (next_feature_string)
        *next_feature_string++ = '\0';
      if (*curr_feature_string == '^')
        {
          curr_feature_string++;
          clear = true;
        }
      for (i = 0; i < X86_TUNE_LAST; i++)
        {
          if (!strcmp (curr_feature_string, ix86_tune_feature_names[i]))
            {
              ix86_tune_features[i] = !clear;
              if (dump)
                fprintf (stderr, "Explicitly %s feature %s\n",
                         clear ? "clear" : "set", ix86_tune_feature_names[i]);
              break;
            }
        }
      if (i == X86_TUNE_LAST)
        error ("Unknown parameter to option -mtune-ctrl: %s",
               clear ? curr_feature_string - 1 : curr_feature_string);
      curr_feature_string = next_feature_string;
    }
  while (curr_feature_string);
  free (orig);
}

/* Helper function to set ix86_tune_features. IX86_TUNE is the
   processor type.  */

static void
set_ix86_tune_features (enum processor_type ix86_tune, bool dump)
{
  unsigned int ix86_tune_mask = 1u << ix86_tune;
  int i;

  for (i = 0; i < X86_TUNE_LAST; ++i)
    {
      if (ix86_tune_no_default)
        ix86_tune_features[i] = 0;
      else
        ix86_tune_features[i] = !!(initial_ix86_tune_features[i] & ix86_tune_mask);
    }

  if (dump)
    {
      fprintf (stderr, "List of x86 specific tuning parameter names:\n");
      for (i = 0; i < X86_TUNE_LAST; i++)
        fprintf (stderr, "%s : %s\n", ix86_tune_feature_names[i],
                 ix86_tune_features[i] ? "on" : "off");
    }

  parse_mtune_ctrl_str (dump);
}


/* Default align_* from the processor table.  */

static void
ix86_default_align (struct gcc_options *opts)
{
  if (opts->x_align_loops == 0)
    {
      opts->x_align_loops = processor_target_table[ix86_tune].align_loop;
      align_loops_max_skip = processor_target_table[ix86_tune].align_loop_max_skip;
    }
  if (opts->x_align_jumps == 0)
    {
      opts->x_align_jumps = processor_target_table[ix86_tune].align_jump;
      align_jumps_max_skip = processor_target_table[ix86_tune].align_jump_max_skip;
    }
  if (opts->x_align_functions == 0)
    {
      opts->x_align_functions = processor_target_table[ix86_tune].align_func;
    }
}

/* Implement TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE hook.  */

static void
ix86_override_options_after_change (void)
{
  ix86_default_align (&global_options);
}

/* Override various settings based on options.  If MAIN_ARGS_P, the
   options are from the command line, otherwise they are from
   attributes.  Return true if there's an error related to march
   option.  */

static bool
ix86_option_override_internal (bool main_args_p,
			       struct gcc_options *opts,
			       struct gcc_options *opts_set)
{
  int i;
  unsigned int ix86_arch_mask;
  const bool ix86_tune_specified = (opts->x_ix86_tune_string != NULL);

#define PTA_3DNOW		(HOST_WIDE_INT_1 << 0)
#define PTA_3DNOW_A		(HOST_WIDE_INT_1 << 1)
#define PTA_64BIT		(HOST_WIDE_INT_1 << 2)
#define PTA_ABM			(HOST_WIDE_INT_1 << 3)
#define PTA_AES			(HOST_WIDE_INT_1 << 4)
#define PTA_AVX			(HOST_WIDE_INT_1 << 5)
#define PTA_BMI			(HOST_WIDE_INT_1 << 6)
#define PTA_CX16		(HOST_WIDE_INT_1 << 7)
#define PTA_F16C		(HOST_WIDE_INT_1 << 8)
#define PTA_FMA			(HOST_WIDE_INT_1 << 9)
#define PTA_FMA4		(HOST_WIDE_INT_1 << 10)
#define PTA_FSGSBASE		(HOST_WIDE_INT_1 << 11)
#define PTA_LWP			(HOST_WIDE_INT_1 << 12)
#define PTA_LZCNT		(HOST_WIDE_INT_1 << 13)
#define PTA_MMX			(HOST_WIDE_INT_1 << 14)
#define PTA_MOVBE		(HOST_WIDE_INT_1 << 15)
#define PTA_NO_SAHF		(HOST_WIDE_INT_1 << 16)
#define PTA_PCLMUL		(HOST_WIDE_INT_1 << 17)
#define PTA_POPCNT		(HOST_WIDE_INT_1 << 18)
#define PTA_PREFETCH_SSE	(HOST_WIDE_INT_1 << 19)
#define PTA_RDRND		(HOST_WIDE_INT_1 << 20)
#define PTA_SSE			(HOST_WIDE_INT_1 << 21)
#define PTA_SSE2		(HOST_WIDE_INT_1 << 22)
#define PTA_SSE3		(HOST_WIDE_INT_1 << 23)
#define PTA_SSE4_1		(HOST_WIDE_INT_1 << 24)
#define PTA_SSE4_2		(HOST_WIDE_INT_1 << 25)
#define PTA_SSE4A		(HOST_WIDE_INT_1 << 26)
#define PTA_SSSE3		(HOST_WIDE_INT_1 << 27)
#define PTA_TBM			(HOST_WIDE_INT_1 << 28)
#define PTA_XOP			(HOST_WIDE_INT_1 << 29)
#define PTA_AVX2		(HOST_WIDE_INT_1 << 30)
#define PTA_BMI2		(HOST_WIDE_INT_1 << 31)
#define PTA_RTM			(HOST_WIDE_INT_1 << 32)
#define PTA_HLE			(HOST_WIDE_INT_1 << 33)
#define PTA_PRFCHW		(HOST_WIDE_INT_1 << 34)
#define PTA_RDSEED		(HOST_WIDE_INT_1 << 35)
#define PTA_ADX			(HOST_WIDE_INT_1 << 36)
#define PTA_FXSR		(HOST_WIDE_INT_1 << 37)
#define PTA_XSAVE		(HOST_WIDE_INT_1 << 38)
#define PTA_XSAVEOPT		(HOST_WIDE_INT_1 << 39)
#define PTA_AVX512F		(HOST_WIDE_INT_1 << 40)
#define PTA_AVX512ER		(HOST_WIDE_INT_1 << 41)
#define PTA_AVX512PF		(HOST_WIDE_INT_1 << 42)
#define PTA_AVX512CD		(HOST_WIDE_INT_1 << 43)
#define PTA_MPX			(HOST_WIDE_INT_1 << 44)
#define PTA_SHA			(HOST_WIDE_INT_1 << 45)
#define PTA_PREFETCHWT1		(HOST_WIDE_INT_1 << 46)
#define PTA_CLFLUSHOPT		(HOST_WIDE_INT_1 << 47)
#define PTA_XSAVEC		(HOST_WIDE_INT_1 << 48)
#define PTA_XSAVES		(HOST_WIDE_INT_1 << 49)
#define PTA_AVX512DQ		(HOST_WIDE_INT_1 << 50)
#define PTA_AVX512BW		(HOST_WIDE_INT_1 << 51)
#define PTA_AVX512VL		(HOST_WIDE_INT_1 << 52)
#define PTA_AVX512IFMA		(HOST_WIDE_INT_1 << 53)
#define PTA_AVX512VBMI		(HOST_WIDE_INT_1 << 54)
#define PTA_CLWB		(HOST_WIDE_INT_1 << 55)
#define PTA_MWAITX		(HOST_WIDE_INT_1 << 56)
#define PTA_CLZERO		(HOST_WIDE_INT_1 << 57)
#define PTA_NO_80387		(HOST_WIDE_INT_1 << 58)
#define PTA_PKU			(HOST_WIDE_INT_1 << 59)
#define PTA_AVX5124VNNIW	(HOST_WIDE_INT_1 << 60)
#define PTA_AVX5124FMAPS	(HOST_WIDE_INT_1 << 61)
#define PTA_AVX512VPOPCNTDQ	(HOST_WIDE_INT_1 << 62)
#define PTA_SGX			(HOST_WIDE_INT_1 << 63)

#define PTA_CORE2 \
  (PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSSE3 \
   | PTA_CX16 | PTA_FXSR)
#define PTA_NEHALEM \
  (PTA_CORE2 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_POPCNT)
#define PTA_WESTMERE \
  (PTA_NEHALEM | PTA_AES | PTA_PCLMUL)
#define PTA_SANDYBRIDGE \
  (PTA_WESTMERE | PTA_AVX | PTA_XSAVE | PTA_XSAVEOPT)
#define PTA_IVYBRIDGE \
  (PTA_SANDYBRIDGE | PTA_FSGSBASE | PTA_RDRND | PTA_F16C)
#define PTA_HASWELL \
  (PTA_IVYBRIDGE | PTA_AVX2 | PTA_BMI | PTA_BMI2 | PTA_LZCNT \
   | PTA_FMA | PTA_MOVBE | PTA_HLE)
#define PTA_BROADWELL \
  (PTA_HASWELL | PTA_ADX | PTA_PRFCHW | PTA_RDSEED)
#define PTA_SKYLAKE \
  (PTA_BROADWELL | PTA_CLFLUSHOPT | PTA_XSAVEC | PTA_XSAVES)
#define PTA_SKYLAKE_AVX512 \
  (PTA_SKYLAKE | PTA_AVX512F | PTA_AVX512CD | PTA_AVX512VL \
   | PTA_AVX512BW | PTA_AVX512DQ | PTA_PKU)
#define PTA_KNL \
  (PTA_BROADWELL | PTA_AVX512PF | PTA_AVX512ER | PTA_AVX512F | PTA_AVX512CD)
#define PTA_BONNELL \
  (PTA_CORE2 | PTA_MOVBE)
#define PTA_SILVERMONT \
  (PTA_WESTMERE | PTA_MOVBE)

/* if this reaches 64, need to widen struct pta flags below */

  static struct pta
    {
      const char *const name;		/* processor name or nickname.  */
      const enum processor_type processor;
      const enum attr_cpu schedule;
      const unsigned HOST_WIDE_INT flags;
    }
  const processor_alias_table[] =
    {
      {"i386", PROCESSOR_I386, CPU_NONE, 0},
      {"i486", PROCESSOR_I486, CPU_NONE, 0},
      {"i586", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
      {"pentium", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
      {"lakemont", PROCESSOR_LAKEMONT, CPU_PENTIUM, PTA_NO_80387},
      {"pentium-mmx", PROCESSOR_PENTIUM, CPU_PENTIUM, PTA_MMX},
      {"winchip-c6", PROCESSOR_I486, CPU_NONE, PTA_MMX},
      {"winchip2", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW},
      {"c3", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW},
      {"samuel-2", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW},
      {"c3-2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
	PTA_MMX | PTA_SSE | PTA_FXSR},
      {"nehemiah", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
        PTA_MMX | PTA_SSE | PTA_FXSR},
      {"c7", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
        PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR},
      {"esther", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
        PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR},
      {"i686", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
      {"pentiumpro", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
      {"pentium2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, PTA_MMX | PTA_FXSR},
      {"pentium3", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
	PTA_MMX | PTA_SSE | PTA_FXSR},
      {"pentium3m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
	PTA_MMX | PTA_SSE | PTA_FXSR},
      {"pentium-m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
	PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_FXSR},
      {"pentium4", PROCESSOR_PENTIUM4, CPU_NONE,
	PTA_MMX |PTA_SSE | PTA_SSE2 | PTA_FXSR},
      {"pentium4m", PROCESSOR_PENTIUM4, CPU_NONE,
	PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_FXSR},
      {"prescott", PROCESSOR_NOCONA, CPU_NONE,
	PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR},
      {"nocona", PROCESSOR_NOCONA, CPU_NONE,
	PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
	| PTA_CX16 | PTA_NO_SAHF | PTA_FXSR},
      {"core2", PROCESSOR_CORE2, CPU_CORE2, PTA_CORE2},
      {"nehalem", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_NEHALEM},
      {"corei7", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_NEHALEM},
      {"westmere", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_WESTMERE},
      {"sandybridge", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
	PTA_SANDYBRIDGE},
      {"corei7-avx", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
	PTA_SANDYBRIDGE},
      {"ivybridge", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
	PTA_IVYBRIDGE},
      {"core-avx-i", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
	PTA_IVYBRIDGE},
      {"haswell", PROCESSOR_HASWELL, CPU_HASWELL, PTA_HASWELL},
      {"core-avx2", PROCESSOR_HASWELL, CPU_HASWELL, PTA_HASWELL},
      {"broadwell", PROCESSOR_HASWELL, CPU_HASWELL, PTA_BROADWELL},
      {"skylake", PROCESSOR_HASWELL, CPU_HASWELL, PTA_SKYLAKE},
      {"skylake-avx512", PROCESSOR_HASWELL, CPU_HASWELL, PTA_SKYLAKE_AVX512},
      {"bonnell", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL},
      {"atom", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL},
      {"silvermont", PROCESSOR_SILVERMONT, CPU_SLM, PTA_SILVERMONT},
      {"slm", PROCESSOR_SILVERMONT, CPU_SLM, PTA_SILVERMONT},
      {"knl", PROCESSOR_KNL, CPU_SLM, PTA_KNL},
      {"intel", PROCESSOR_INTEL, CPU_SLM, PTA_NEHALEM},
      {"geode", PROCESSOR_GEODE, CPU_GEODE,
	PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE},
      {"k6", PROCESSOR_K6, CPU_K6, PTA_MMX},
      {"k6-2", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW},
      {"k6-3", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW},
      {"athlon", PROCESSOR_ATHLON, CPU_ATHLON,
	PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE},
      {"athlon-tbird", PROCESSOR_ATHLON, CPU_ATHLON,
	PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE},
      {"athlon-4", PROCESSOR_ATHLON, CPU_ATHLON,
	PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_FXSR},
      {"athlon-xp", PROCESSOR_ATHLON, CPU_ATHLON,
	PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_FXSR},
      {"athlon-mp", PROCESSOR_ATHLON, CPU_ATHLON,
	PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_FXSR},
      {"x86-64", PROCESSOR_K8, CPU_K8,
	PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR},
      {"eden-x2", PROCESSOR_K8, CPU_K8,
        PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR},
      {"nano", PROCESSOR_K8, CPU_K8,
        PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
        | PTA_SSSE3 | PTA_FXSR},
      {"nano-1000", PROCESSOR_K8, CPU_K8,
        PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
        | PTA_SSSE3 | PTA_FXSR},
      {"nano-2000", PROCESSOR_K8, CPU_K8,
        PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
        | PTA_SSSE3 | PTA_FXSR},
      {"nano-3000", PROCESSOR_K8, CPU_K8,
        PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
        | PTA_SSSE3 | PTA_SSE4_1 | PTA_FXSR},
      {"nano-x2", PROCESSOR_K8, CPU_K8,
        PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
        | PTA_SSSE3 | PTA_SSE4_1 | PTA_FXSR},
      {"eden-x4", PROCESSOR_K8, CPU_K8,
        PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
        | PTA_SSSE3 | PTA_SSE4_1 | PTA_FXSR},
      {"nano-x4", PROCESSOR_K8, CPU_K8,
        PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
        | PTA_SSSE3 | PTA_SSE4_1 | PTA_FXSR},
      {"k8", PROCESSOR_K8, CPU_K8,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
	| PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR},
      {"k8-sse3", PROCESSOR_K8, CPU_K8,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
	| PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_FXSR},
      {"opteron", PROCESSOR_K8, CPU_K8,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
	| PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR},
      {"opteron-sse3", PROCESSOR_K8, CPU_K8,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
	| PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_FXSR},
      {"athlon64", PROCESSOR_K8, CPU_K8,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
	| PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR},
      {"athlon64-sse3", PROCESSOR_K8, CPU_K8,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
	| PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_FXSR},
      {"athlon-fx", PROCESSOR_K8, CPU_K8,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
	| PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR},
      {"amdfam10", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2
	| PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_PRFCHW | PTA_FXSR},
      {"barcelona", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
	PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2
	| PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_PRFCHW | PTA_FXSR},
      {"bdver1", PROCESSOR_BDVER1, CPU_BDVER1,
	PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
	| PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
	| PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
	| PTA_XOP | PTA_LWP | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE},
      {"bdver2", PROCESSOR_BDVER2, CPU_BDVER2,
	PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
	| PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
	| PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
	| PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C
	| PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE},
      {"bdver3", PROCESSOR_BDVER3, CPU_BDVER3,
	PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
	| PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
	| PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
	| PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C
	| PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE 
	| PTA_XSAVEOPT | PTA_FSGSBASE},
      {"bdver4", PROCESSOR_BDVER4, CPU_BDVER4,
	PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
	| PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
	| PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_AVX2 
	| PTA_FMA4 | PTA_XOP | PTA_LWP | PTA_BMI | PTA_BMI2 
	| PTA_TBM | PTA_F16C | PTA_FMA | PTA_PRFCHW | PTA_FXSR 
	| PTA_XSAVE | PTA_XSAVEOPT | PTA_FSGSBASE | PTA_RDRND
	| PTA_MOVBE | PTA_MWAITX},
      {"znver1", PROCESSOR_ZNVER1, CPU_ZNVER1,
	PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
	| PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
	| PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_AVX2
	| PTA_BMI | PTA_BMI2 | PTA_F16C | PTA_FMA | PTA_PRFCHW
	| PTA_FXSR | PTA_XSAVE | PTA_XSAVEOPT | PTA_FSGSBASE
	| PTA_RDRND | PTA_MOVBE | PTA_MWAITX | PTA_ADX | PTA_RDSEED
	| PTA_CLZERO | PTA_CLFLUSHOPT | PTA_XSAVEC | PTA_XSAVES
	| PTA_SHA | PTA_LZCNT | PTA_POPCNT},
      {"btver1", PROCESSOR_BTVER1, CPU_GENERIC,
	PTA_64BIT | PTA_MMX |  PTA_SSE  | PTA_SSE2 | PTA_SSE3
	| PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16 | PTA_PRFCHW
	| PTA_FXSR | PTA_XSAVE},
      {"btver2", PROCESSOR_BTVER2, CPU_BTVER2,
	PTA_64BIT | PTA_MMX |  PTA_SSE  | PTA_SSE2 | PTA_SSE3
	| PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16 | PTA_SSE4_1
	| PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX
	| PTA_BMI | PTA_F16C | PTA_MOVBE | PTA_PRFCHW
	| PTA_FXSR | PTA_XSAVE | PTA_XSAVEOPT},

      {"generic", PROCESSOR_GENERIC, CPU_GENERIC,
	PTA_64BIT
	| PTA_HLE /* flags are only used for -march switch.  */ },
    };

  /* -mrecip options.  */
  static struct
    {
      const char *string;           /* option name */
      unsigned int mask;            /* mask bits to set */
    }
  const recip_options[] =
    {
      { "all",       RECIP_MASK_ALL },
      { "none",      RECIP_MASK_NONE },
      { "div",       RECIP_MASK_DIV },
      { "sqrt",      RECIP_MASK_SQRT },
      { "vec-div",   RECIP_MASK_VEC_DIV },
      { "vec-sqrt",  RECIP_MASK_VEC_SQRT },
    };

  int const pta_size = ARRAY_SIZE (processor_alias_table);

  /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
     TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false.  */
  if (TARGET_64BIT_DEFAULT && !TARGET_64BIT_P (opts->x_ix86_isa_flags))
    opts->x_ix86_isa_flags &= ~(OPTION_MASK_ABI_64 | OPTION_MASK_ABI_X32);
#ifdef TARGET_BI_ARCH
  else
    {
#if TARGET_BI_ARCH == 1
      /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
	 is on and OPTION_MASK_ABI_X32 is off.  We turn off
	 OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
	 -mx32.  */
      if (TARGET_X32_P (opts->x_ix86_isa_flags))
	opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_64;
#else
      /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
	 on and OPTION_MASK_ABI_64 is off.  We turn off
	 OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
	 -m64 or OPTION_MASK_CODE16 is turned on by -m16.  */
      if (TARGET_LP64_P (opts->x_ix86_isa_flags)
	  || TARGET_16BIT_P (opts->x_ix86_isa_flags))
	opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
#endif
      if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
	  && TARGET_IAMCU_P (opts->x_target_flags))
	sorry ("Intel MCU psABI isn%'t supported in %s mode",
	       TARGET_X32_P (opts->x_ix86_isa_flags) ? "x32" : "64-bit");
    }
#endif

  if (TARGET_X32_P (opts->x_ix86_isa_flags))
    {
      /* Always turn on OPTION_MASK_ISA_64BIT and turn off
	 OPTION_MASK_ABI_64 for TARGET_X32.  */
      opts->x_ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
      opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_64;
    }
  else if (TARGET_16BIT_P (opts->x_ix86_isa_flags))
    opts->x_ix86_isa_flags &= ~(OPTION_MASK_ISA_64BIT
				| OPTION_MASK_ABI_X32
				| OPTION_MASK_ABI_64);
  else if (TARGET_LP64_P (opts->x_ix86_isa_flags))
    {
      /* Always turn on OPTION_MASK_ISA_64BIT and turn off
	 OPTION_MASK_ABI_X32 for TARGET_LP64.  */
      opts->x_ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
      opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
    }

#ifdef SUBTARGET_OVERRIDE_OPTIONS
  SUBTARGET_OVERRIDE_OPTIONS;
#endif

#ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
  SUBSUBTARGET_OVERRIDE_OPTIONS;
#endif

  /* -fPIC is the default for x86_64.  */
  if (TARGET_MACHO && TARGET_64BIT_P (opts->x_ix86_isa_flags))
    opts->x_flag_pic = 2;

  /* Need to check -mtune=generic first.  */
  if (opts->x_ix86_tune_string)
    {
      /* As special support for cross compilers we read -mtune=native
	     as -mtune=generic.  With native compilers we won't see the
	     -mtune=native, as it was changed by the driver.  */
      if (!strcmp (opts->x_ix86_tune_string, "native"))
	{
	  opts->x_ix86_tune_string = "generic";
	}
      else if (!strcmp (opts->x_ix86_tune_string, "x86-64"))
        warning (OPT_Wdeprecated,
		 main_args_p
		 ? G_("%<-mtune=x86-64%> is deprecated; use %<-mtune=k8%> "
		      "or %<-mtune=generic%> instead as appropriate")
		 : G_("%<target(\"tune=x86-64\")%> is deprecated; use "
		      "%<target(\"tune=k8\")%> or %<target(\"tune=generic\")%>"
		      " instead as appropriate"));
    }
  else
    {
      if (opts->x_ix86_arch_string)
	opts->x_ix86_tune_string = opts->x_ix86_arch_string;
      if (!opts->x_ix86_tune_string)
	{
	  opts->x_ix86_tune_string
	    = processor_target_table[TARGET_CPU_DEFAULT].name;
	  ix86_tune_defaulted = 1;
	}

      /* opts->x_ix86_tune_string is set to opts->x_ix86_arch_string
	 or defaulted.  We need to use a sensible tune option.  */
      if (!strcmp (opts->x_ix86_tune_string, "x86-64"))
	{
	  opts->x_ix86_tune_string = "generic";
	}
    }

  if (opts->x_ix86_stringop_alg == rep_prefix_8_byte
      && !TARGET_64BIT_P (opts->x_ix86_isa_flags))
    {
      /* rep; movq isn't available in 32-bit code.  */
      error ("-mstringop-strategy=rep_8byte not supported for 32-bit code");
      opts->x_ix86_stringop_alg = no_stringop;
    }

  if (!opts->x_ix86_arch_string)
    opts->x_ix86_arch_string
      = TARGET_64BIT_P (opts->x_ix86_isa_flags)
	? "x86-64" : SUBTARGET32_DEFAULT_CPU;
  else
    ix86_arch_specified = 1;

  if (opts_set->x_ix86_pmode)
    {
      if ((TARGET_LP64_P (opts->x_ix86_isa_flags)
	   && opts->x_ix86_pmode == PMODE_SI)
	  || (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
	       && opts->x_ix86_pmode == PMODE_DI))
	error ("address mode %qs not supported in the %s bit mode",
	       TARGET_64BIT_P (opts->x_ix86_isa_flags) ? "short" : "long",
	       TARGET_64BIT_P (opts->x_ix86_isa_flags) ? "64" : "32");
    }
  else
    opts->x_ix86_pmode = TARGET_LP64_P (opts->x_ix86_isa_flags)
			 ? PMODE_DI : PMODE_SI;

  if (!opts_set->x_ix86_abi)
    opts->x_ix86_abi = DEFAULT_ABI;

  /* For targets using ms ABI enable ms-extensions, if not
     explicit turned off.  For non-ms ABI we turn off this
     option.  */
  if (!opts_set->x_flag_ms_extensions)
    opts->x_flag_ms_extensions = (MS_ABI == DEFAULT_ABI);

  if (opts_set->x_ix86_cmodel)
    {
      switch (opts->x_ix86_cmodel)
	{
	case CM_SMALL:
	case CM_SMALL_PIC:
	  if (opts->x_flag_pic)
	    opts->x_ix86_cmodel = CM_SMALL_PIC;
	  if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
	    error ("code model %qs not supported in the %s bit mode",
		   "small", "32");
	  break;

	case CM_MEDIUM:
	case CM_MEDIUM_PIC:
	  if (opts->x_flag_pic)
	    opts->x_ix86_cmodel = CM_MEDIUM_PIC;
	  if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
	    error ("code model %qs not supported in the %s bit mode",
		   "medium", "32");
	  else if (TARGET_X32_P (opts->x_ix86_isa_flags))
	    error ("code model %qs not supported in x32 mode",
		   "medium");
	  break;

	case CM_LARGE:
	case CM_LARGE_PIC:
	  if (opts->x_flag_pic)
	    opts->x_ix86_cmodel = CM_LARGE_PIC;
	  if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
	    error ("code model %qs not supported in the %s bit mode",
		   "large", "32");
	  else if (TARGET_X32_P (opts->x_ix86_isa_flags))
	    error ("code model %qs not supported in x32 mode",
		   "large");
	  break;

	case CM_32:
	  if (opts->x_flag_pic)
	    error ("code model %s does not support PIC mode", "32");
	  if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
	    error ("code model %qs not supported in the %s bit mode",
		   "32", "64");
	  break;

	case CM_KERNEL:
	  if (opts->x_flag_pic)
	    {
	      error ("code model %s does not support PIC mode", "kernel");
	      opts->x_ix86_cmodel = CM_32;
	    }
	  if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
	    error ("code model %qs not supported in the %s bit mode",
		   "kernel", "32");
	  break;

	default:
	  gcc_unreachable ();
	}
    }
  else
    {
      /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
	 use of rip-relative addressing.  This eliminates fixups that
	 would otherwise be needed if this object is to be placed in a
	 DLL, and is essentially just as efficient as direct addressing.  */
      if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
	  && (TARGET_RDOS || TARGET_PECOFF))
	opts->x_ix86_cmodel = CM_MEDIUM_PIC, opts->x_flag_pic = 1;
      else if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
	opts->x_ix86_cmodel = opts->x_flag_pic ? CM_SMALL_PIC : CM_SMALL;
      else
	opts->x_ix86_cmodel = CM_32;
    }
  if (TARGET_MACHO && opts->x_ix86_asm_dialect == ASM_INTEL)
    {
      error ("-masm=intel not supported in this configuration");
      opts->x_ix86_asm_dialect = ASM_ATT;
    }
  if ((TARGET_64BIT_P (opts->x_ix86_isa_flags) != 0)
      != ((opts->x_ix86_isa_flags & OPTION_MASK_ISA_64BIT) != 0))
    sorry ("%i-bit mode not compiled in",
	   (opts->x_ix86_isa_flags & OPTION_MASK_ISA_64BIT) ? 64 : 32);

  for (i = 0; i < pta_size; i++)
    if (! strcmp (opts->x_ix86_arch_string, processor_alias_table[i].name))
      {
	if (!strcmp (opts->x_ix86_arch_string, "generic"))
	  {
	    error (main_args_p
		   ? G_("%<generic%> CPU can be used only for %<-mtune=%> "
			"switch")
		   : G_("%<generic%> CPU can be used only for "
			"%<target(\"tune=\")%> attribute"));
	    return false;
	  }
	else if (!strcmp (opts->x_ix86_arch_string, "intel"))
	  {
	    error (main_args_p
		   ? G_("%<intel%> CPU can be used only for %<-mtune=%> "
			"switch")
		   : G_("%<intel%> CPU can be used only for "
			"%<target(\"tune=\")%> attribute"));
	    return false;
	  }

	if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
	    && !(processor_alias_table[i].flags & PTA_64BIT))
	  {
	    error ("CPU you selected does not support x86-64 "
		   "instruction set");
	    return false;
	  }

	ix86_schedule = processor_alias_table[i].schedule;
	ix86_arch = processor_alias_table[i].processor;
	/* Default cpu tuning to the architecture.  */
	ix86_tune = ix86_arch;

	if (processor_alias_table[i].flags & PTA_MMX
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MMX))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MMX;
	if (processor_alias_table[i].flags & PTA_3DNOW
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW;
	if (processor_alias_table[i].flags & PTA_3DNOW_A
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW_A))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_A;
	if (processor_alias_table[i].flags & PTA_SSE
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE;
	if (processor_alias_table[i].flags & PTA_SSE2
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE2))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE2;
	if (processor_alias_table[i].flags & PTA_SSE3
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE3))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE3;
	if (processor_alias_table[i].flags & PTA_SSSE3
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSSE3))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSSE3;
	if (processor_alias_table[i].flags & PTA_SSE4_1
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_1))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1;
	if (processor_alias_table[i].flags & PTA_SSE4_2
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2;
	if (processor_alias_table[i].flags & PTA_AVX
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX;
	if (processor_alias_table[i].flags & PTA_AVX2
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX2))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX2;
	if (processor_alias_table[i].flags & PTA_FMA
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA;
	if (processor_alias_table[i].flags & PTA_SSE4A
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4A;
	if (processor_alias_table[i].flags & PTA_FMA4
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA4))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA4;
	if (processor_alias_table[i].flags & PTA_XOP
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XOP))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XOP;
	if (processor_alias_table[i].flags & PTA_LWP
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LWP))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LWP;
	if (processor_alias_table[i].flags & PTA_ABM
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ABM;
	if (processor_alias_table[i].flags & PTA_BMI
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI;
	if (processor_alias_table[i].flags & (PTA_LZCNT | PTA_ABM)
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LZCNT))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LZCNT;
	if (processor_alias_table[i].flags & PTA_TBM
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_TBM))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_TBM;
	if (processor_alias_table[i].flags & PTA_BMI2
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI2))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI2;
	if (processor_alias_table[i].flags & PTA_CX16
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CX16;
	if (processor_alias_table[i].flags & (PTA_POPCNT | PTA_ABM)
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_POPCNT))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_POPCNT;
	if (!(TARGET_64BIT_P (opts->x_ix86_isa_flags)
	    && (processor_alias_table[i].flags & PTA_NO_SAHF))
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SAHF))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SAHF;
	if (processor_alias_table[i].flags & PTA_MOVBE
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MOVBE))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MOVBE;
	if (processor_alias_table[i].flags & PTA_AES
	    && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AES))
	  ix86_isa_flags |= OPTION_MASK_ISA_AES;
	if (processor_alias_table[i].flags & PTA_SHA
	    && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SHA))
	  ix86_isa_flags |= OPTION_MASK_ISA_SHA;
	if (processor_alias_table[i].flags & PTA_PCLMUL
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL;
	if (processor_alias_table[i].flags & PTA_FSGSBASE
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FSGSBASE))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FSGSBASE;
	if (processor_alias_table[i].flags & PTA_RDRND
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDRND))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDRND;
	if (processor_alias_table[i].flags & PTA_F16C
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_F16C))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_F16C;
	if (processor_alias_table[i].flags & PTA_RTM
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RTM))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RTM;
	if (processor_alias_table[i].flags & PTA_HLE
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_HLE))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_HLE;
	if (processor_alias_table[i].flags & PTA_PRFCHW
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PRFCHW))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PRFCHW;
	if (processor_alias_table[i].flags & PTA_RDSEED
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDSEED))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDSEED;
	if (processor_alias_table[i].flags & PTA_ADX
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ADX))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ADX;
	if (processor_alias_table[i].flags & PTA_FXSR
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FXSR))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FXSR;
	if (processor_alias_table[i].flags & PTA_XSAVE
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVE))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVE;
	if (processor_alias_table[i].flags & PTA_XSAVEOPT
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEOPT))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEOPT;
	if (processor_alias_table[i].flags & PTA_AVX512F
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512F))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512F;
	if (processor_alias_table[i].flags & PTA_AVX512ER
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512ER))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512ER;
	if (processor_alias_table[i].flags & PTA_AVX512PF
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512PF))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512PF;
	if (processor_alias_table[i].flags & PTA_AVX512CD
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512CD))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512CD;
	if (processor_alias_table[i].flags & PTA_PREFETCHWT1
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PREFETCHWT1))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PREFETCHWT1;
	if (processor_alias_table[i].flags & PTA_CLWB
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLWB))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLWB;
	if (processor_alias_table[i].flags & PTA_CLFLUSHOPT
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLFLUSHOPT))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLFLUSHOPT;
	if (processor_alias_table[i].flags & PTA_CLZERO
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLZERO))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLZERO;
	if (processor_alias_table[i].flags & PTA_XSAVEC
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEC))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEC;
	if (processor_alias_table[i].flags & PTA_XSAVES
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVES))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVES;
	if (processor_alias_table[i].flags & PTA_AVX512DQ
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512DQ))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512DQ;
	if (processor_alias_table[i].flags & PTA_AVX512BW
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512BW))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512BW;
	if (processor_alias_table[i].flags & PTA_AVX512VL
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VL))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VL;
        if (processor_alias_table[i].flags & PTA_MPX
            && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MPX))
          opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MPX;
	if (processor_alias_table[i].flags & PTA_AVX512VBMI
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VBMI))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VBMI;
	if (processor_alias_table[i].flags & PTA_AVX512IFMA
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512IFMA))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512IFMA;

	if (processor_alias_table[i].flags & PTA_AVX5124VNNIW
	    && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_AVX5124VNNIW))
	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_AVX5124VNNIW;
	if (processor_alias_table[i].flags & PTA_AVX5124FMAPS
	    && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_AVX5124FMAPS))
	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_AVX5124FMAPS;
	if (processor_alias_table[i].flags & PTA_AVX512VPOPCNTDQ
	    && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_AVX512VPOPCNTDQ))
	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_AVX512VPOPCNTDQ;
	if (processor_alias_table[i].flags & PTA_SGX
	    && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_SGX))
	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_SGX;

	if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE))
	  x86_prefetch_sse = true;
	if (processor_alias_table[i].flags & PTA_MWAITX
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MWAITX))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MWAITX;
	if (processor_alias_table[i].flags & PTA_PKU
	    && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PKU))
	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PKU;

	/* Don't enable x87 instructions if only
	   general registers are allowed.  */
	if (!(opts_set->x_ix86_target_flags & OPTION_MASK_GENERAL_REGS_ONLY)
	    && !(opts_set->x_target_flags & MASK_80387))
	  {
	    if (processor_alias_table[i].flags & PTA_NO_80387)
	      opts->x_target_flags &= ~MASK_80387;
	    else
	      opts->x_target_flags |= MASK_80387;
	  }
	break;
      }

  if (TARGET_X32 && (opts->x_ix86_isa_flags & OPTION_MASK_ISA_MPX))
    error ("Intel MPX does not support x32");

  if (TARGET_X32 && (ix86_isa_flags & OPTION_MASK_ISA_MPX))
    error ("Intel MPX does not support x32");

  if (i == pta_size)
    {
      error (main_args_p
	     ? G_("bad value (%qs) for %<-march=%> switch")
	     : G_("bad value (%qs) for %<target(\"arch=\")%> attribute"),
	     opts->x_ix86_arch_string);

      auto_vec <const char *> candidates;
      for (i = 0; i < pta_size; i++)
	if (strcmp (processor_alias_table[i].name, "generic")
	    && strcmp (processor_alias_table[i].name, "intel")
	    && (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
		|| (processor_alias_table[i].flags & PTA_64BIT)))
	  candidates.safe_push (processor_alias_table[i].name);

      char *s;
      const char *hint
	= candidates_list_and_hint (opts->x_ix86_arch_string, s, candidates);
      if (hint)
	inform (input_location,
		main_args_p
		? G_("valid arguments to %<-march=%> switch are: "
		     "%s; did you mean %qs?")
		: G_("valid arguments to %<target(\"arch=\")%> attribute are: "
		     "%s; did you mean %qs?"), s, hint);
      else
	inform (input_location,
		main_args_p
		? G_("valid arguments to %<-march=%> switch are: %s")
		: G_("valid arguments to %<target(\"arch=\")%> attribute "
		     "are: %s"), s);
      XDELETEVEC (s);
    }

  ix86_arch_mask = 1u << ix86_arch;
  for (i = 0; i < X86_ARCH_LAST; ++i)
    ix86_arch_features[i] = !!(initial_ix86_arch_features[i] & ix86_arch_mask);

  for (i = 0; i < pta_size; i++)
    if (! strcmp (opts->x_ix86_tune_string, processor_alias_table[i].name))
      {
	ix86_schedule = processor_alias_table[i].schedule;
	ix86_tune = processor_alias_table[i].processor;
	if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
	  {
	    if (!(processor_alias_table[i].flags & PTA_64BIT))
	      {
		if (ix86_tune_defaulted)
		  {
		    opts->x_ix86_tune_string = "x86-64";
		    for (i = 0; i < pta_size; i++)
		      if (! strcmp (opts->x_ix86_tune_string,
				    processor_alias_table[i].name))
			break;
		    ix86_schedule = processor_alias_table[i].schedule;
		    ix86_tune = processor_alias_table[i].processor;
		  }
		else
		  error ("CPU you selected does not support x86-64 "
			 "instruction set");
	      }
	  }
	/* Intel CPUs have always interpreted SSE prefetch instructions as
	   NOPs; so, we can enable SSE prefetch instructions even when
	   -mtune (rather than -march) points us to a processor that has them.
	   However, the VIA C3 gives a SIGILL, so we only do that for i686 and
	   higher processors.  */
	if (TARGET_CMOV
	    && (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)))
	  x86_prefetch_sse = true;
	break;
      }

  if (ix86_tune_specified && i == pta_size)
    {
      error (main_args_p
	     ? G_("bad value (%qs) for %<-mtune=%> switch")
	     : G_("bad value (%qs) for %<target(\"tune=\")%> attribute"),
	     opts->x_ix86_tune_string);

      auto_vec <const char *> candidates;
      for (i = 0; i < pta_size; i++)
	if (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
	    || (processor_alias_table[i].flags & PTA_64BIT))
	  candidates.safe_push (processor_alias_table[i].name);

      char *s;
      const char *hint
	= candidates_list_and_hint (opts->x_ix86_tune_string, s, candidates);
      if (hint)
	inform (input_location,
		main_args_p
		? G_("valid arguments to %<-mtune=%> switch are: "
		     "%s; did you mean %qs?")
		: G_("valid arguments to %<target(\"tune=\")%> attribute are: "
		     "%s; did you mean %qs?"), s, hint);
      else
	inform (input_location,
		main_args_p
		? G_("valid arguments to %<-mtune=%> switch are: %s")
		: G_("valid arguments to %<target(\"tune=\")%> attribute "
		     "are: %s"), s);
      XDELETEVEC (s);
    }

  set_ix86_tune_features (ix86_tune, opts->x_ix86_dump_tunes);

#ifndef USE_IX86_FRAME_POINTER
#define USE_IX86_FRAME_POINTER 0
#endif

#ifndef USE_X86_64_FRAME_POINTER
#define USE_X86_64_FRAME_POINTER 0
#endif

  /* Set the default values for switches whose default depends on TARGET_64BIT
     in case they weren't overwritten by command line options.  */
  if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
    {
      if (opts->x_optimize >= 1 && !opts_set->x_flag_omit_frame_pointer)
	opts->x_flag_omit_frame_pointer = !USE_X86_64_FRAME_POINTER;
      if (opts->x_flag_asynchronous_unwind_tables
	  && !opts_set->x_flag_unwind_tables
	  && TARGET_64BIT_MS_ABI)
	opts->x_flag_unwind_tables = 1;
      if (opts->x_flag_asynchronous_unwind_tables == 2)
	opts->x_flag_unwind_tables
	  = opts->x_flag_asynchronous_unwind_tables = 1;
      if (opts->x_flag_pcc_struct_return == 2)
	opts->x_flag_pcc_struct_return = 0;
    }
  else
    {
      if (opts->x_optimize >= 1 && !opts_set->x_flag_omit_frame_pointer)
	opts->x_flag_omit_frame_pointer
	  = !(USE_IX86_FRAME_POINTER || opts->x_optimize_size);
      if (opts->x_flag_asynchronous_unwind_tables == 2)
	opts->x_flag_asynchronous_unwind_tables = !USE_IX86_FRAME_POINTER;
      if (opts->x_flag_pcc_struct_return == 2)
	{
	  /* Intel MCU psABI specifies that -freg-struct-return should
	     be on.  Instead of setting DEFAULT_PCC_STRUCT_RETURN to 1,
	     we check -miamcu so that -freg-struct-return is always
	     turned on if -miamcu is used.  */
	  if (TARGET_IAMCU_P (opts->x_target_flags))
	    opts->x_flag_pcc_struct_return = 0;
	  else
	    opts->x_flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN;
	}
    }

  ix86_tune_cost = processor_target_table[ix86_tune].cost;
  /* TODO: ix86_cost should be chosen at instruction or function granuality
     so for cold code we use size_cost even in !optimize_size compilation.  */
  if (opts->x_optimize_size)
    ix86_cost = &ix86_size_cost;
  else
    ix86_cost = ix86_tune_cost;

  /* Arrange to set up i386_stack_locals for all functions.  */
  init_machine_status = ix86_init_machine_status;

  /* Validate -mregparm= value.  */
  if (opts_set->x_ix86_regparm)
    {
      if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
	warning (0, "-mregparm is ignored in 64-bit mode");
      else if (TARGET_IAMCU_P (opts->x_target_flags))
	warning (0, "-mregparm is ignored for Intel MCU psABI");
      if (opts->x_ix86_regparm > REGPARM_MAX)
	{
	  error ("-mregparm=%d is not between 0 and %d",
		 opts->x_ix86_regparm, REGPARM_MAX);
	  opts->x_ix86_regparm = 0;
	}
    }
  if (TARGET_IAMCU_P (opts->x_target_flags)
      || TARGET_64BIT_P (opts->x_ix86_isa_flags))
    opts->x_ix86_regparm = REGPARM_MAX;

  /* Default align_* from the processor table.  */
  ix86_default_align (opts);

  /* Provide default for -mbranch-cost= value.  */
  if (!opts_set->x_ix86_branch_cost)
    opts->x_ix86_branch_cost = ix86_tune_cost->branch_cost;

  if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
    {
      opts->x_target_flags
	|= TARGET_SUBTARGET64_DEFAULT & ~opts_set->x_target_flags;

      /* Enable by default the SSE and MMX builtins.  Do allow the user to
	 explicitly disable any of these.  In particular, disabling SSE and
	 MMX for kernel code is extremely useful.  */
      if (!ix86_arch_specified)
      opts->x_ix86_isa_flags
	|= ((OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_MMX
	     | TARGET_SUBTARGET64_ISA_DEFAULT)
            & ~opts->x_ix86_isa_flags_explicit);

      if (TARGET_RTD_P (opts->x_target_flags))
	warning (0,
		 main_args_p
		 ? G_("%<-mrtd%> is ignored in 64bit mode")
		 : G_("%<target(\"rtd\")%> is ignored in 64bit mode"));
    }
  else
    {
      opts->x_target_flags
	|= TARGET_SUBTARGET32_DEFAULT & ~opts_set->x_target_flags;

      if (!ix86_arch_specified)
        opts->x_ix86_isa_flags
	  |= TARGET_SUBTARGET32_ISA_DEFAULT & ~opts->x_ix86_isa_flags_explicit;

      /* i386 ABI does not specify red zone.  It still makes sense to use it
         when programmer takes care to stack from being destroyed.  */
      if (!(opts_set->x_target_flags & MASK_NO_RED_ZONE))
        opts->x_target_flags |= MASK_NO_RED_ZONE;
    }

  /* Keep nonleaf frame pointers.  */
  if (opts->x_flag_omit_frame_pointer)
    opts->x_target_flags &= ~MASK_OMIT_LEAF_FRAME_POINTER;
  else if (TARGET_OMIT_LEAF_FRAME_POINTER_P (opts->x_target_flags))
    opts->x_flag_omit_frame_pointer = 1;

  /* If we're doing fast math, we don't care about comparison order
     wrt NaNs.  This lets us use a shorter comparison sequence.  */
  if (opts->x_flag_finite_math_only)
    opts->x_target_flags &= ~MASK_IEEE_FP;

  /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
     since the insns won't need emulation.  */
  if (ix86_tune_features [X86_TUNE_ALWAYS_FANCY_MATH_387])
    opts->x_target_flags &= ~MASK_NO_FANCY_MATH_387;

  /* Likewise, if the target doesn't have a 387, or we've specified
     software floating point, don't use 387 inline intrinsics.  */
  if (!TARGET_80387_P (opts->x_target_flags))
    opts->x_target_flags |= MASK_NO_FANCY_MATH_387;

  /* Turn on MMX builtins for -msse.  */
  if (TARGET_SSE_P (opts->x_ix86_isa_flags))
    opts->x_ix86_isa_flags
      |= OPTION_MASK_ISA_MMX & ~opts->x_ix86_isa_flags_explicit;

  /* Enable SSE prefetch.  */
  if (TARGET_SSE_P (opts->x_ix86_isa_flags)
      || (TARGET_PRFCHW_P (opts->x_ix86_isa_flags)
	  && !TARGET_3DNOW_P (opts->x_ix86_isa_flags))
      || TARGET_PREFETCHWT1_P (opts->x_ix86_isa_flags))
    x86_prefetch_sse = true;

  /* Enable popcnt instruction for -msse4.2 or -mabm.  */
  if (TARGET_SSE4_2_P (opts->x_ix86_isa_flags)
      || TARGET_ABM_P (opts->x_ix86_isa_flags))
    opts->x_ix86_isa_flags
      |= OPTION_MASK_ISA_POPCNT & ~opts->x_ix86_isa_flags_explicit;

  /* Enable lzcnt instruction for -mabm.  */
  if (TARGET_ABM_P(opts->x_ix86_isa_flags))
    opts->x_ix86_isa_flags
      |= OPTION_MASK_ISA_LZCNT & ~opts->x_ix86_isa_flags_explicit;

  /* Validate -mpreferred-stack-boundary= value or default it to
     PREFERRED_STACK_BOUNDARY_DEFAULT.  */
  ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT;
  if (opts_set->x_ix86_preferred_stack_boundary_arg)
    {
      int min = TARGET_64BIT_P (opts->x_ix86_isa_flags)? 3 : 2;
      int max = TARGET_SEH ? 4 : 12;

      if (opts->x_ix86_preferred_stack_boundary_arg < min
	  || opts->x_ix86_preferred_stack_boundary_arg > max)
	{
	  if (min == max)
	    error ("-mpreferred-stack-boundary is not supported "
		   "for this target");
	  else
	    error ("-mpreferred-stack-boundary=%d is not between %d and %d",
		   opts->x_ix86_preferred_stack_boundary_arg, min, max);
	}
      else
	ix86_preferred_stack_boundary
	  = (1 << opts->x_ix86_preferred_stack_boundary_arg) * BITS_PER_UNIT;
    }

  /* Set the default value for -mstackrealign.  */
  if (opts->x_ix86_force_align_arg_pointer == -1)
    opts->x_ix86_force_align_arg_pointer = STACK_REALIGN_DEFAULT;

  ix86_default_incoming_stack_boundary = PREFERRED_STACK_BOUNDARY;

  /* Validate -mincoming-stack-boundary= value or default it to
     MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY.  */
  ix86_incoming_stack_boundary = ix86_default_incoming_stack_boundary;
  if (opts_set->x_ix86_incoming_stack_boundary_arg)
    {
      int min = TARGET_64BIT_P (opts->x_ix86_isa_flags) ? 3 : 2;

      if (opts->x_ix86_incoming_stack_boundary_arg < min
	  || opts->x_ix86_incoming_stack_boundary_arg > 12)
	error ("-mincoming-stack-boundary=%d is not between %d and 12",
	       opts->x_ix86_incoming_stack_boundary_arg, min);
      else
	{
	  ix86_user_incoming_stack_boundary
	    = (1 << opts->x_ix86_incoming_stack_boundary_arg) * BITS_PER_UNIT;
	  ix86_incoming_stack_boundary
	    = ix86_user_incoming_stack_boundary;
	}
    }

#ifndef NO_PROFILE_COUNTERS
  if (flag_nop_mcount)
    error ("-mnop-mcount is not compatible with this target");
#endif
  if (flag_nop_mcount && flag_pic)
    error ("-mnop-mcount is not implemented for -fPIC");

  /* Accept -msseregparm only if at least SSE support is enabled.  */
  if (TARGET_SSEREGPARM_P (opts->x_target_flags)
      && ! TARGET_SSE_P (opts->x_ix86_isa_flags))
    error (main_args_p
	   ? G_("%<-msseregparm%> used without SSE enabled")
	   : G_("%<target(\"sseregparm\")%> used without SSE enabled"));

  if (opts_set->x_ix86_fpmath)
    {
      if (opts->x_ix86_fpmath & FPMATH_SSE)
	{
	  if (!TARGET_SSE_P (opts->x_ix86_isa_flags))
	    {
	      if (TARGET_80387_P (opts->x_target_flags))
		{
		  warning (0, "SSE instruction set disabled, using 387 arithmetics");
		  opts->x_ix86_fpmath = FPMATH_387;
		}
	    }
	  else if ((opts->x_ix86_fpmath & FPMATH_387)
		   && !TARGET_80387_P (opts->x_target_flags))
	    {
	      warning (0, "387 instruction set disabled, using SSE arithmetics");
	      opts->x_ix86_fpmath = FPMATH_SSE;
	    }
	}
    }
  /* For all chips supporting SSE2, -mfpmath=sse performs better than
     fpmath=387.  The second is however default at many targets since the
     extra 80bit precision of temporaries is considered to be part of ABI.
     Overwrite the default at least for -ffast-math. 
     TODO: -mfpmath=both seems to produce same performing code with bit
     smaller binaries.  It is however not clear if register allocation is
     ready for this setting.
     Also -mfpmath=387 is overall a lot more compact (bout 4-5%) than SSE
     codegen.  We may switch to 387 with -ffast-math for size optimized
     functions. */
  else if (fast_math_flags_set_p (&global_options)
	   && TARGET_SSE2_P (opts->x_ix86_isa_flags))
    opts->x_ix86_fpmath = FPMATH_SSE;
  else
    opts->x_ix86_fpmath = TARGET_FPMATH_DEFAULT_P (opts->x_ix86_isa_flags);

  /* Use external vectorized library in vectorizing intrinsics.  */
  if (opts_set->x_ix86_veclibabi_type)
    switch (opts->x_ix86_veclibabi_type)
      {
      case ix86_veclibabi_type_svml:
	ix86_veclib_handler = ix86_veclibabi_svml;
	break;

      case ix86_veclibabi_type_acml:
	ix86_veclib_handler = ix86_veclibabi_acml;
	break;

      default:
	gcc_unreachable ();
      }

  if (ix86_tune_features [X86_TUNE_ACCUMULATE_OUTGOING_ARGS]
      && !(opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
    opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;

  /* If stack probes are required, the space used for large function
     arguments on the stack must also be probed, so enable
     -maccumulate-outgoing-args so this happens in the prologue.  */
  if (TARGET_STACK_PROBE_P (opts->x_target_flags)
      && !(opts->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
    {
      if (opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS)
	warning (0,
		 main_args_p
		 ? G_("stack probing requires %<-maccumulate-outgoing-args%> "
		      "for correctness")
		 : G_("stack probing requires "
		      "%<target(\"accumulate-outgoing-args\")%> for "
		      "correctness"));
      opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
    }

  /* Stack realignment without -maccumulate-outgoing-args requires %ebp,
     so enable -maccumulate-outgoing-args when %ebp is fixed.  */
  if (fixed_regs[BP_REG]
      && !(opts->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
    {
      if (opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS)
	warning (0,
		 main_args_p
		 ? G_("fixed ebp register requires "
		      "%<-maccumulate-outgoing-args%>")
		 : G_("fixed ebp register requires "
		      "%<target(\"accumulate-outgoing-args\")%>"));
      opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
    }

  /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix.  */
  {
    char *p;
    ASM_GENERATE_INTERNAL_LABEL (internal_label_prefix, "LX", 0);
    p = strchr (internal_label_prefix, 'X');
    internal_label_prefix_len = p - internal_label_prefix;
    *p = '\0';
  }

  /* When scheduling description is not available, disable scheduler pass
     so it won't slow down the compilation and make x87 code slower.  */
  if (!TARGET_SCHEDULE)
    opts->x_flag_schedule_insns_after_reload = opts->x_flag_schedule_insns = 0;

  maybe_set_param_value (PARAM_SIMULTANEOUS_PREFETCHES,
			 ix86_tune_cost->simultaneous_prefetches,
			 opts->x_param_values,
			 opts_set->x_param_values);
  maybe_set_param_value (PARAM_L1_CACHE_LINE_SIZE,
			 ix86_tune_cost->prefetch_block,
			 opts->x_param_values,
			 opts_set->x_param_values);
  maybe_set_param_value (PARAM_L1_CACHE_SIZE,
			 ix86_tune_cost->l1_cache_size,
			 opts->x_param_values,
			 opts_set->x_param_values);
  maybe_set_param_value (PARAM_L2_CACHE_SIZE,
			 ix86_tune_cost->l2_cache_size,
			 opts->x_param_values,
			 opts_set->x_param_values);

  /* Restrict number of if-converted SET insns to 1.  */
  if (TARGET_ONE_IF_CONV_INSN)
    maybe_set_param_value (PARAM_MAX_RTL_IF_CONVERSION_INSNS,
			   1,
			   opts->x_param_values,
			   opts_set->x_param_values);

  /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful.  */
  if (opts->x_flag_prefetch_loop_arrays < 0
      && HAVE_prefetch
      && (opts->x_optimize >= 3 || opts->x_flag_profile_use)
      && !opts->x_optimize_size
      && TARGET_SOFTWARE_PREFETCHING_BENEFICIAL)
    opts->x_flag_prefetch_loop_arrays = 1;

  /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
     can be opts->x_optimized to ap = __builtin_next_arg (0).  */
  if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && !opts->x_flag_split_stack)
    targetm.expand_builtin_va_start = NULL;

  if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
    {
      ix86_gen_leave = gen_leave_rex64;
      if (Pmode == DImode)
	{
	  ix86_gen_tls_global_dynamic_64 = gen_tls_global_dynamic_64_di;
	  ix86_gen_tls_local_dynamic_base_64
	    = gen_tls_local_dynamic_base_64_di;
	}
      else
	{
	  ix86_gen_tls_global_dynamic_64 = gen_tls_global_dynamic_64_si;
	  ix86_gen_tls_local_dynamic_base_64
	    = gen_tls_local_dynamic_base_64_si;
	}
    }
  else
    ix86_gen_leave = gen_leave;

  if (Pmode == DImode)
    {
      ix86_gen_add3 = gen_adddi3;
      ix86_gen_sub3 = gen_subdi3;
      ix86_gen_sub3_carry = gen_subdi3_carry;
      ix86_gen_one_cmpl2 = gen_one_cmpldi2;
      ix86_gen_andsp = gen_anddi3;
      ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_di;
      ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probedi;
      ix86_gen_probe_stack_range = gen_probe_stack_rangedi;
      ix86_gen_monitor = gen_sse3_monitor_di;
      ix86_gen_monitorx = gen_monitorx_di;
      ix86_gen_clzero = gen_clzero_di;
    }
  else
    {
      ix86_gen_add3 = gen_addsi3;
      ix86_gen_sub3 = gen_subsi3;
      ix86_gen_sub3_carry = gen_subsi3_carry;
      ix86_gen_one_cmpl2 = gen_one_cmplsi2;
      ix86_gen_andsp = gen_andsi3;
      ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_si;
      ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probesi;
      ix86_gen_probe_stack_range = gen_probe_stack_rangesi;
      ix86_gen_monitor = gen_sse3_monitor_si;
      ix86_gen_monitorx = gen_monitorx_si;
      ix86_gen_clzero = gen_clzero_si;
    }

#ifdef USE_IX86_CLD
  /* Use -mcld by default for 32-bit code if configured with --enable-cld.  */
  if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
    opts->x_target_flags |= MASK_CLD & ~opts_set->x_target_flags;
#endif

  if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && opts->x_flag_pic)
    {
      if (opts->x_flag_fentry > 0)
        sorry ("-mfentry isn%'t supported for 32-bit in combination "
	       "with -fpic");
      opts->x_flag_fentry = 0;
    }
  else if (TARGET_SEH)
    {
      if (opts->x_flag_fentry == 0)
	sorry ("-mno-fentry isn%'t compatible with SEH");
      opts->x_flag_fentry = 1;
    }
  else if (opts->x_flag_fentry < 0)
   {
#if defined(PROFILE_BEFORE_PROLOGUE)
     opts->x_flag_fentry = 1;
#else
     opts->x_flag_fentry = 0;
#endif
   }

  if (!(opts_set->x_target_flags & MASK_VZEROUPPER))
    opts->x_target_flags |= MASK_VZEROUPPER;
  if (!(opts_set->x_target_flags & MASK_STV))
    opts->x_target_flags |= MASK_STV;
  /* Disable STV if -mpreferred-stack-boundary={2,3} or
     -mincoming-stack-boundary={2,3} or -mstackrealign - the needed
     stack realignment will be extra cost the pass doesn't take into
     account and the pass can't realign the stack.  */
  if (ix86_preferred_stack_boundary < 128
      || ix86_incoming_stack_boundary < 128
      || opts->x_ix86_force_align_arg_pointer)
    opts->x_target_flags &= ~MASK_STV;
  if (!ix86_tune_features[X86_TUNE_AVX256_UNALIGNED_LOAD_OPTIMAL]
      && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_LOAD))
    opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_LOAD;
  if (!ix86_tune_features[X86_TUNE_AVX256_UNALIGNED_STORE_OPTIMAL]
      && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_STORE))
    opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_STORE;
  /* Enable 128-bit AVX instruction generation
     for the auto-vectorizer.  */
  if (TARGET_AVX128_OPTIMAL
      && !(opts_set->x_target_flags & MASK_PREFER_AVX128))
    opts->x_target_flags |= MASK_PREFER_AVX128;

  if (opts->x_ix86_recip_name)
    {
      char *p = ASTRDUP (opts->x_ix86_recip_name);
      char *q;
      unsigned int mask, i;
      bool invert;

      while ((q = strtok (p, ",")) != NULL)
	{
	  p = NULL;
	  if (*q == '!')
	    {
	      invert = true;
	      q++;
	    }
	  else
	    invert = false;

	  if (!strcmp (q, "default"))
	    mask = RECIP_MASK_ALL;
	  else
	    {
	      for (i = 0; i < ARRAY_SIZE (recip_options); i++)
		if (!strcmp (q, recip_options[i].string))
		  {
		    mask = recip_options[i].mask;
		    break;
		  }

	      if (i == ARRAY_SIZE (recip_options))
		{
		  error ("unknown option for -mrecip=%s", q);
		  invert = false;
		  mask = RECIP_MASK_NONE;
		}
	    }

	  opts->x_recip_mask_explicit |= mask;
	  if (invert)
	    opts->x_recip_mask &= ~mask;
	  else
	    opts->x_recip_mask |= mask;
	}
    }

  if (TARGET_RECIP_P (opts->x_target_flags))
    opts->x_recip_mask |= RECIP_MASK_ALL & ~opts->x_recip_mask_explicit;
  else if (opts_set->x_target_flags & MASK_RECIP)
    opts->x_recip_mask &= ~(RECIP_MASK_ALL & ~opts->x_recip_mask_explicit);

  /* Default long double to 64-bit for 32-bit Bionic and to __float128
     for 64-bit Bionic.  Also default long double to 64-bit for Intel
     MCU psABI.  */
  if ((TARGET_HAS_BIONIC || TARGET_IAMCU)
      && !(opts_set->x_target_flags
	   & (MASK_LONG_DOUBLE_64 | MASK_LONG_DOUBLE_128)))
    opts->x_target_flags |= (TARGET_64BIT
			     ? MASK_LONG_DOUBLE_128
			     : MASK_LONG_DOUBLE_64);

  /* Only one of them can be active.  */
  gcc_assert ((opts->x_target_flags & MASK_LONG_DOUBLE_64) == 0
	      || (opts->x_target_flags & MASK_LONG_DOUBLE_128) == 0);

  /* Save the initial options in case the user does function specific
     options.  */
  if (main_args_p)
    target_option_default_node = target_option_current_node
      = build_target_option_node (opts);

  /* Handle stack protector */
  if (!opts_set->x_ix86_stack_protector_guard)
    opts->x_ix86_stack_protector_guard
      = TARGET_HAS_BIONIC ? SSP_GLOBAL : SSP_TLS;

  /* Handle -mmemcpy-strategy= and -mmemset-strategy=  */
  if (opts->x_ix86_tune_memcpy_strategy)
    {
      char *str = xstrdup (opts->x_ix86_tune_memcpy_strategy);
      ix86_parse_stringop_strategy_string (str, false);
      free (str);
    }

  if (opts->x_ix86_tune_memset_strategy)
    {
      char *str = xstrdup (opts->x_ix86_tune_memset_strategy);
      ix86_parse_stringop_strategy_string (str, true);
      free (str);
    }

  return true;
}

/* Implement the TARGET_OPTION_OVERRIDE hook.  */

static void
ix86_option_override (void)
{
  ix86_option_override_internal (true, &global_options, &global_options_set);
}

/* Implement the TARGET_OFFLOAD_OPTIONS hook.  */
static char *
ix86_offload_options (void)
{
  if (TARGET_LP64)
    return xstrdup ("-foffload-abi=lp64");
  return xstrdup ("-foffload-abi=ilp32");
}

/* Update register usage after having seen the compiler flags.  */

static void
ix86_conditional_register_usage (void)
{
  int i, c_mask;

  /* If there are no caller-saved registers, preserve all registers.
     except fixed_regs and registers used for function return value
     since aggregate_value_p checks call_used_regs[regno] on return
     value.  */
  if (cfun && cfun->machine->no_caller_saved_registers)
    for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
      if (!fixed_regs[i] && !ix86_function_value_regno_p (i))
	call_used_regs[i] = 0;

  /* For 32-bit targets, squash the REX registers.  */
  if (! TARGET_64BIT)
    {
      for (i = FIRST_REX_INT_REG; i <= LAST_REX_INT_REG; i++)
	fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
      for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
	fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
      for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
	fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
    }

  /*  See the definition of CALL_USED_REGISTERS in i386.h.  */
  c_mask = CALL_USED_REGISTERS_MASK (TARGET_64BIT_MS_ABI);
  
  CLEAR_HARD_REG_SET (reg_class_contents[(int)CLOBBERED_REGS]);

  for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
    {
      /* Set/reset conditionally defined registers from
	 CALL_USED_REGISTERS initializer.  */
      if (call_used_regs[i] > 1)
	call_used_regs[i] = !!(call_used_regs[i] & c_mask);

      /* Calculate registers of CLOBBERED_REGS register set
	 as call used registers from GENERAL_REGS register set.  */
      if (TEST_HARD_REG_BIT (reg_class_contents[(int)GENERAL_REGS], i)
	  && call_used_regs[i])
	SET_HARD_REG_BIT (reg_class_contents[(int)CLOBBERED_REGS], i);
    }

  /* If MMX is disabled, squash the registers.  */
  if (! TARGET_MMX)
    for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
      if (TEST_HARD_REG_BIT (reg_class_contents[(int)MMX_REGS], i))
	fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";

  /* If SSE is disabled, squash the registers.  */
  if (! TARGET_SSE)
    for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
      if (TEST_HARD_REG_BIT (reg_class_contents[(int)SSE_REGS], i))
	fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";

  /* If the FPU is disabled, squash the registers.  */
  if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
    for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
      if (TEST_HARD_REG_BIT (reg_class_contents[(int)FLOAT_REGS], i))
	fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";

  /* If AVX512F is disabled, squash the registers.  */
  if (! TARGET_AVX512F)
    {
      for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
	fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";

      for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
	fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
    }

  /* If MPX is disabled, squash the registers.  */
  if (! TARGET_MPX)
    for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++)
      fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
}


/* Save the current options */

static void
ix86_function_specific_save (struct cl_target_option *ptr,
			     struct gcc_options *opts)
{
  ptr->arch = ix86_arch;
  ptr->schedule = ix86_schedule;
  ptr->prefetch_sse = x86_prefetch_sse;
  ptr->tune = ix86_tune;
  ptr->branch_cost = ix86_branch_cost;
  ptr->tune_defaulted = ix86_tune_defaulted;
  ptr->arch_specified = ix86_arch_specified;
  ptr->x_ix86_isa_flags_explicit = opts->x_ix86_isa_flags_explicit;
  ptr->x_ix86_isa_flags2_explicit = opts->x_ix86_isa_flags2_explicit;
  ptr->x_recip_mask_explicit = opts->x_recip_mask_explicit;
  ptr->x_ix86_arch_string = opts->x_ix86_arch_string;
  ptr->x_ix86_tune_string = opts->x_ix86_tune_string;
  ptr->x_ix86_cmodel = opts->x_ix86_cmodel;
  ptr->x_ix86_abi = opts->x_ix86_abi;
  ptr->x_ix86_asm_dialect = opts->x_ix86_asm_dialect;
  ptr->x_ix86_branch_cost = opts->x_ix86_branch_cost;
  ptr->x_ix86_dump_tunes = opts->x_ix86_dump_tunes;
  ptr->x_ix86_force_align_arg_pointer = opts->x_ix86_force_align_arg_pointer;
  ptr->x_ix86_force_drap = opts->x_ix86_force_drap;
  ptr->x_ix86_incoming_stack_boundary_arg = opts->x_ix86_incoming_stack_boundary_arg;
  ptr->x_ix86_pmode = opts->x_ix86_pmode;
  ptr->x_ix86_preferred_stack_boundary_arg = opts->x_ix86_preferred_stack_boundary_arg;
  ptr->x_ix86_recip_name = opts->x_ix86_recip_name;
  ptr->x_ix86_regparm = opts->x_ix86_regparm;
  ptr->x_ix86_section_threshold = opts->x_ix86_section_threshold;
  ptr->x_ix86_sse2avx = opts->x_ix86_sse2avx;
  ptr->x_ix86_stack_protector_guard = opts->x_ix86_stack_protector_guard;
  ptr->x_ix86_stringop_alg = opts->x_ix86_stringop_alg;
  ptr->x_ix86_tls_dialect = opts->x_ix86_tls_dialect;
  ptr->x_ix86_tune_ctrl_string = opts->x_ix86_tune_ctrl_string;
  ptr->x_ix86_tune_memcpy_strategy = opts->x_ix86_tune_memcpy_strategy;
  ptr->x_ix86_tune_memset_strategy = opts->x_ix86_tune_memset_strategy;
  ptr->x_ix86_tune_no_default = opts->x_ix86_tune_no_default;
  ptr->x_ix86_veclibabi_type = opts->x_ix86_veclibabi_type;

  /* The fields are char but the variables are not; make sure the
     values fit in the fields.  */
  gcc_assert (ptr->arch == ix86_arch);
  gcc_assert (ptr->schedule == ix86_schedule);
  gcc_assert (ptr->tune == ix86_tune);
  gcc_assert (ptr->branch_cost == ix86_branch_cost);
}

/* Restore the current options */

static void
ix86_function_specific_restore (struct gcc_options *opts,
				struct cl_target_option *ptr)
{
  enum processor_type old_tune = ix86_tune;
  enum processor_type old_arch = ix86_arch;
  unsigned int ix86_arch_mask;
  int i;

  /* We don't change -fPIC.  */
  opts->x_flag_pic = flag_pic;

  ix86_arch = (enum processor_type) ptr->arch;
  ix86_schedule = (enum attr_cpu) ptr->schedule;
  ix86_tune = (enum processor_type) ptr->tune;
  x86_prefetch_sse = ptr->prefetch_sse;
  opts->x_ix86_branch_cost = ptr->branch_cost;
  ix86_tune_defaulted = ptr->tune_defaulted;
  ix86_arch_specified = ptr->arch_specified;
  opts->x_ix86_isa_flags_explicit = ptr->x_ix86_isa_flags_explicit;
  opts->x_ix86_isa_flags2_explicit = ptr->x_ix86_isa_flags2_explicit;
  opts->x_recip_mask_explicit = ptr->x_recip_mask_explicit;
  opts->x_ix86_arch_string = ptr->x_ix86_arch_string;
  opts->x_ix86_tune_string = ptr->x_ix86_tune_string;
  opts->x_ix86_cmodel = ptr->x_ix86_cmodel;
  opts->x_ix86_abi = ptr->x_ix86_abi;
  opts->x_ix86_asm_dialect = ptr->x_ix86_asm_dialect;
  opts->x_ix86_branch_cost = ptr->x_ix86_branch_cost;
  opts->x_ix86_dump_tunes = ptr->x_ix86_dump_tunes;
  opts->x_ix86_force_align_arg_pointer = ptr->x_ix86_force_align_arg_pointer;
  opts->x_ix86_force_drap = ptr->x_ix86_force_drap;
  opts->x_ix86_incoming_stack_boundary_arg = ptr->x_ix86_incoming_stack_boundary_arg;
  opts->x_ix86_pmode = ptr->x_ix86_pmode;
  opts->x_ix86_preferred_stack_boundary_arg = ptr->x_ix86_preferred_stack_boundary_arg;
  opts->x_ix86_recip_name = ptr->x_ix86_recip_name;
  opts->x_ix86_regparm = ptr->x_ix86_regparm;
  opts->x_ix86_section_threshold = ptr->x_ix86_section_threshold;
  opts->x_ix86_sse2avx = ptr->x_ix86_sse2avx;
  opts->x_ix86_stack_protector_guard = ptr->x_ix86_stack_protector_guard;
  opts->x_ix86_stringop_alg = ptr->x_ix86_stringop_alg;
  opts->x_ix86_tls_dialect = ptr->x_ix86_tls_dialect;
  opts->x_ix86_tune_ctrl_string = ptr->x_ix86_tune_ctrl_string;
  opts->x_ix86_tune_memcpy_strategy = ptr->x_ix86_tune_memcpy_strategy;
  opts->x_ix86_tune_memset_strategy = ptr->x_ix86_tune_memset_strategy;
  opts->x_ix86_tune_no_default = ptr->x_ix86_tune_no_default;
  opts->x_ix86_veclibabi_type = ptr->x_ix86_veclibabi_type;
  ix86_tune_cost = processor_target_table[ix86_tune].cost;
  /* TODO: ix86_cost should be chosen at instruction or function granuality
     so for cold code we use size_cost even in !optimize_size compilation.  */
  if (opts->x_optimize_size)
    ix86_cost = &ix86_size_cost;
  else
    ix86_cost = ix86_tune_cost;

  /* Recreate the arch feature tests if the arch changed */
  if (old_arch != ix86_arch)
    {
      ix86_arch_mask = 1u << ix86_arch;
      for (i = 0; i < X86_ARCH_LAST; ++i)
	ix86_arch_features[i]
	  = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
    }

  /* Recreate the tune optimization tests */
  if (old_tune != ix86_tune)
    set_ix86_tune_features (ix86_tune, false);
}

/* Adjust target options after streaming them in.  This is mainly about
   reconciling them with global options.  */

static void
ix86_function_specific_post_stream_in (struct cl_target_option *ptr)
{
  /* flag_pic is a global option, but ix86_cmodel is target saved option
     partly computed from flag_pic.  If flag_pic is on, adjust x_ix86_cmodel
     for PIC, or error out.  */
  if (flag_pic)
    switch (ptr->x_ix86_cmodel)
      {
      case CM_SMALL:
	ptr->x_ix86_cmodel = CM_SMALL_PIC;
	break;

      case CM_MEDIUM:
	ptr->x_ix86_cmodel = CM_MEDIUM_PIC;
	break;

      case CM_LARGE:
	ptr->x_ix86_cmodel = CM_LARGE_PIC;
	break;

      case CM_KERNEL:
	error ("code model %s does not support PIC mode", "kernel");
	break;

      default:
	break;
      }
  else
    switch (ptr->x_ix86_cmodel)
      {
      case CM_SMALL_PIC:
	ptr->x_ix86_cmodel = CM_SMALL;
	break;

      case CM_MEDIUM_PIC:
	ptr->x_ix86_cmodel = CM_MEDIUM;
	break;

      case CM_LARGE_PIC:
	ptr->x_ix86_cmodel = CM_LARGE;
	break;

      default:
	break;
      }
}

/* Print the current options */

static void
ix86_function_specific_print (FILE *file, int indent,
			      struct cl_target_option *ptr)
{
  char *target_string
    = ix86_target_string (ptr->x_ix86_isa_flags, ptr->x_ix86_isa_flags2,
			  ptr->x_target_flags, ptr->x_ix86_target_flags,
			  NULL, NULL, ptr->x_ix86_fpmath, false);

  gcc_assert (ptr->arch < PROCESSOR_max);
  fprintf (file, "%*sarch = %d (%s)\n",
	   indent, "",
	   ptr->arch, processor_target_table[ptr->arch].name);

  gcc_assert (ptr->tune < PROCESSOR_max);
  fprintf (file, "%*stune = %d (%s)\n",
	   indent, "",
	   ptr->tune, processor_target_table[ptr->tune].name);

  fprintf (file, "%*sbranch_cost = %d\n", indent, "", ptr->branch_cost);

  if (target_string)
    {
      fprintf (file, "%*s%s\n", indent, "", target_string);
      free (target_string);
    }
}


/* Inner function to process the attribute((target(...))), take an argument and
   set the current options from the argument. If we have a list, recursively go
   over the list.  */

static bool
ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
				     struct gcc_options *opts,
				     struct gcc_options *opts_set,
				     struct gcc_options *enum_opts_set)
{
  char *next_optstr;
  bool ret = true;

#define IX86_ATTR_ISA(S,O)   { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
#define IX86_ATTR_STR(S,O)   { S, sizeof (S)-1, ix86_opt_str, O, 0 }
#define IX86_ATTR_ENUM(S,O)  { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
#define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
#define IX86_ATTR_NO(S,O,M)  { S, sizeof (S)-1, ix86_opt_no,  O, M }

  enum ix86_opt_type
  {
    ix86_opt_unknown,
    ix86_opt_yes,
    ix86_opt_no,
    ix86_opt_str,
    ix86_opt_enum,
    ix86_opt_isa
  };

  static const struct
  {
    const char *string;
    size_t len;
    enum ix86_opt_type type;
    int opt;
    int mask;
  } attrs[] = {
    /* isa options */
    IX86_ATTR_ISA ("sgx",	OPT_msgx),
    IX86_ATTR_ISA ("avx5124fmaps", OPT_mavx5124fmaps),
    IX86_ATTR_ISA ("avx5124vnniw", OPT_mavx5124vnniw),
    IX86_ATTR_ISA ("avx512vpopcntdq", OPT_mavx512vpopcntdq),

    IX86_ATTR_ISA ("avx512vbmi", OPT_mavx512vbmi),
    IX86_ATTR_ISA ("avx512ifma", OPT_mavx512ifma),
    IX86_ATTR_ISA ("avx512vl",	OPT_mavx512vl),
    IX86_ATTR_ISA ("avx512bw",	OPT_mavx512bw),
    IX86_ATTR_ISA ("avx512dq",	OPT_mavx512dq),
    IX86_ATTR_ISA ("avx512er",	OPT_mavx512er),
    IX86_ATTR_ISA ("avx512pf",	OPT_mavx512pf),
    IX86_ATTR_ISA ("avx512cd",	OPT_mavx512cd),
    IX86_ATTR_ISA ("avx512f",	OPT_mavx512f),
    IX86_ATTR_ISA ("avx2",	OPT_mavx2),
    IX86_ATTR_ISA ("fma",	OPT_mfma),
    IX86_ATTR_ISA ("xop",	OPT_mxop),
    IX86_ATTR_ISA ("fma4",	OPT_mfma4),
    IX86_ATTR_ISA ("f16c",	OPT_mf16c),
    IX86_ATTR_ISA ("avx",	OPT_mavx),
    IX86_ATTR_ISA ("sse4",	OPT_msse4),
    IX86_ATTR_ISA ("sse4.2",	OPT_msse4_2),
    IX86_ATTR_ISA ("sse4.1",	OPT_msse4_1),
    IX86_ATTR_ISA ("sse4a",	OPT_msse4a),
    IX86_ATTR_ISA ("ssse3",	OPT_mssse3),
    IX86_ATTR_ISA ("sse3",	OPT_msse3),
    IX86_ATTR_ISA ("aes",	OPT_maes),
    IX86_ATTR_ISA ("sha",	OPT_msha),
    IX86_ATTR_ISA ("pclmul",	OPT_mpclmul),
    IX86_ATTR_ISA ("sse2",	OPT_msse2),
    IX86_ATTR_ISA ("sse",	OPT_msse),
    IX86_ATTR_ISA ("3dnowa",	OPT_m3dnowa),
    IX86_ATTR_ISA ("3dnow",	OPT_m3dnow),
    IX86_ATTR_ISA ("mmx",	OPT_mmmx),
    IX86_ATTR_ISA ("rtm",	OPT_mrtm),
    IX86_ATTR_ISA ("prfchw",	OPT_mprfchw),
    IX86_ATTR_ISA ("rdseed",	OPT_mrdseed),
    IX86_ATTR_ISA ("adx",	OPT_madx),
    IX86_ATTR_ISA ("prefetchwt1", OPT_mprefetchwt1),
    IX86_ATTR_ISA ("clflushopt", OPT_mclflushopt),
    IX86_ATTR_ISA ("xsaves",	OPT_mxsaves),
    IX86_ATTR_ISA ("xsavec",	OPT_mxsavec),
    IX86_ATTR_ISA ("xsaveopt",	OPT_mxsaveopt),
    IX86_ATTR_ISA ("xsave",	OPT_mxsave),
    IX86_ATTR_ISA ("abm",	OPT_mabm),
    IX86_ATTR_ISA ("bmi",	OPT_mbmi),
    IX86_ATTR_ISA ("bmi2",	OPT_mbmi2),
    IX86_ATTR_ISA ("lzcnt",	OPT_mlzcnt),
    IX86_ATTR_ISA ("tbm",	OPT_mtbm),
    IX86_ATTR_ISA ("popcnt",	OPT_mpopcnt),
    IX86_ATTR_ISA ("cx16",	OPT_mcx16),
    IX86_ATTR_ISA ("sahf",	OPT_msahf),
    IX86_ATTR_ISA ("movbe",	OPT_mmovbe),
    IX86_ATTR_ISA ("crc32",	OPT_mcrc32),
    IX86_ATTR_ISA ("fsgsbase",	OPT_mfsgsbase),
    IX86_ATTR_ISA ("rdrnd",	OPT_mrdrnd),
    IX86_ATTR_ISA ("mwaitx",	OPT_mmwaitx),
    IX86_ATTR_ISA ("clzero",	OPT_mclzero),
    IX86_ATTR_ISA ("pku",	OPT_mpku),
    IX86_ATTR_ISA ("lwp",	OPT_mlwp),
    IX86_ATTR_ISA ("hle",	OPT_mhle),
    IX86_ATTR_ISA ("fxsr",	OPT_mfxsr),
    IX86_ATTR_ISA ("mpx",	OPT_mmpx),
    IX86_ATTR_ISA ("clwb",	OPT_mclwb),
    IX86_ATTR_ISA ("rdpid",	OPT_mrdpid),

    /* enum options */
    IX86_ATTR_ENUM ("fpmath=",	OPT_mfpmath_),

    /* string options */
    IX86_ATTR_STR ("arch=",	IX86_FUNCTION_SPECIFIC_ARCH),
    IX86_ATTR_STR ("tune=",	IX86_FUNCTION_SPECIFIC_TUNE),

    /* flag options */
    IX86_ATTR_YES ("cld",
		   OPT_mcld,
		   MASK_CLD),

    IX86_ATTR_NO ("fancy-math-387",
		  OPT_mfancy_math_387,
		  MASK_NO_FANCY_MATH_387),

    IX86_ATTR_YES ("ieee-fp",
		   OPT_mieee_fp,
		   MASK_IEEE_FP),

    IX86_ATTR_YES ("inline-all-stringops",
		   OPT_minline_all_stringops,
		   MASK_INLINE_ALL_STRINGOPS),

    IX86_ATTR_YES ("inline-stringops-dynamically",
		   OPT_minline_stringops_dynamically,
		   MASK_INLINE_STRINGOPS_DYNAMICALLY),

    IX86_ATTR_NO ("align-stringops",
		  OPT_mno_align_stringops,
		  MASK_NO_ALIGN_STRINGOPS),

    IX86_ATTR_YES ("recip",
		   OPT_mrecip,
		   MASK_RECIP),

  };

  /* If this is a list, recurse to get the options.  */
  if (TREE_CODE (args) == TREE_LIST)
    {
      bool ret = true;

      for (; args; args = TREE_CHAIN (args))
	if (TREE_VALUE (args)
	    && !ix86_valid_target_attribute_inner_p (TREE_VALUE (args),
						     p_strings, opts, opts_set,
						     enum_opts_set))
	  ret = false;

      return ret;
    }

  else if (TREE_CODE (args) != STRING_CST)
    {
      error ("attribute %<target%> argument not a string");
      return false;
    }

  /* Handle multiple arguments separated by commas.  */
  next_optstr = ASTRDUP (TREE_STRING_POINTER (args));

  while (next_optstr && *next_optstr != '\0')
    {
      char *p = next_optstr;
      char *orig_p = p;
      char *comma = strchr (next_optstr, ',');
      const char *opt_string;
      size_t len, opt_len;
      int opt;
      bool opt_set_p;
      char ch;
      unsigned i;
      enum ix86_opt_type type = ix86_opt_unknown;
      int mask = 0;

      if (comma)
	{
	  *comma = '\0';
	  len = comma - next_optstr;
	  next_optstr = comma + 1;
	}
      else
	{
	  len = strlen (p);
	  next_optstr = NULL;
	}

      /* Recognize no-xxx.  */
      if (len > 3 && p[0] == 'n' && p[1] == 'o' && p[2] == '-')
	{
	  opt_set_p = false;
	  p += 3;
	  len -= 3;
	}
      else
	opt_set_p = true;

      /* Find the option.  */
      ch = *p;
      opt = N_OPTS;
      for (i = 0; i < ARRAY_SIZE (attrs); i++)
	{
	  type = attrs[i].type;
	  opt_len = attrs[i].len;
	  if (ch == attrs[i].string[0]
	      && ((type != ix86_opt_str && type != ix86_opt_enum)
		  ? len == opt_len
		  : len > opt_len)
	      && memcmp (p, attrs[i].string, opt_len) == 0)
	    {
	      opt = attrs[i].opt;
	      mask = attrs[i].mask;
	      opt_string = attrs[i].string;
	      break;
	    }
	}

      /* Process the option.  */
      if (opt == N_OPTS)
	{
	  error ("attribute(target(\"%s\")) is unknown", orig_p);
	  ret = false;
	}

      else if (type == ix86_opt_isa)
	{
	  struct cl_decoded_option decoded;

	  generate_option (opt, NULL, opt_set_p, CL_TARGET, &decoded);
	  ix86_handle_option (opts, opts_set,
			      &decoded, input_location);
	}

      else if (type == ix86_opt_yes || type == ix86_opt_no)
	{
	  if (type == ix86_opt_no)
	    opt_set_p = !opt_set_p;

	  if (opt_set_p)
	    opts->x_target_flags |= mask;
	  else
	    opts->x_target_flags &= ~mask;
	}

      else if (type == ix86_opt_str)
	{
	  if (p_strings[opt])
	    {
	      error ("option(\"%s\") was already specified", opt_string);
	      ret = false;
	    }
	  else
	    p_strings[opt] = xstrdup (p + opt_len);
	}

      else if (type == ix86_opt_enum)
	{
	  bool arg_ok;
	  int value;

	  arg_ok = opt_enum_arg_to_value (opt, p + opt_len, &value, CL_TARGET);
	  if (arg_ok)
	    set_option (opts, enum_opts_set, opt, value,
			p + opt_len, DK_UNSPECIFIED, input_location,
			global_dc);
	  else
	    {
	      error ("attribute(target(\"%s\")) is unknown", orig_p);
	      ret = false;
	    }
	}

      else
	gcc_unreachable ();
    }

  return ret;
}

/* Release allocated strings.  */
static void
release_options_strings (char **option_strings)
{
  /* Free up memory allocated to hold the strings */
  for (unsigned i = 0; i < IX86_FUNCTION_SPECIFIC_MAX; i++)
    free (option_strings[i]);
}

/* Return a TARGET_OPTION_NODE tree of the target options listed or NULL.  */

tree
ix86_valid_target_attribute_tree (tree args,
				  struct gcc_options *opts,
				  struct gcc_options *opts_set)
{
  const char *orig_arch_string = opts->x_ix86_arch_string;
  const char *orig_tune_string = opts->x_ix86_tune_string;
  enum fpmath_unit orig_fpmath_set = opts_set->x_ix86_fpmath;
  int orig_tune_defaulted = ix86_tune_defaulted;
  int orig_arch_specified = ix86_arch_specified;
  char *option_strings[IX86_FUNCTION_SPECIFIC_MAX] = { NULL, NULL };
  tree t = NULL_TREE;
  struct cl_target_option *def
    = TREE_TARGET_OPTION (target_option_default_node);
  struct gcc_options enum_opts_set;

  memset (&enum_opts_set, 0, sizeof (enum_opts_set));

  /* Process each of the options on the chain.  */
  if (! ix86_valid_target_attribute_inner_p (args, option_strings, opts,
					     opts_set, &enum_opts_set))
    return error_mark_node;

  /* If the changed options are different from the default, rerun
     ix86_option_override_internal, and then save the options away.
     The string options are attribute options, and will be undone
     when we copy the save structure.  */
  if (opts->x_ix86_isa_flags != def->x_ix86_isa_flags
      || opts->x_ix86_isa_flags2 != def->x_ix86_isa_flags2
      || opts->x_target_flags != def->x_target_flags
      || option_strings[IX86_FUNCTION_SPECIFIC_ARCH]
      || option_strings[IX86_FUNCTION_SPECIFIC_TUNE]
      || enum_opts_set.x_ix86_fpmath)
    {
      /* If we are using the default tune= or arch=, undo the string assigned,
	 and use the default.  */
      if (option_strings[IX86_FUNCTION_SPECIFIC_ARCH])
	{
	  opts->x_ix86_arch_string
	    = ggc_strdup (option_strings[IX86_FUNCTION_SPECIFIC_ARCH]);

	  /* If arch= is set,  clear all bits in x_ix86_isa_flags,
	     except for ISA_64BIT, ABI_64, ABI_X32, and CODE16.  */
	  opts->x_ix86_isa_flags &= (OPTION_MASK_ISA_64BIT
				     | OPTION_MASK_ABI_64
				     | OPTION_MASK_ABI_X32
				     | OPTION_MASK_CODE16);
	  opts->x_ix86_isa_flags2 = 0;
	}
      else if (!orig_arch_specified)
	opts->x_ix86_arch_string = NULL;

      if (option_strings[IX86_FUNCTION_SPECIFIC_TUNE])
	opts->x_ix86_tune_string
	  = ggc_strdup (option_strings[IX86_FUNCTION_SPECIFIC_TUNE]);
      else if (orig_tune_defaulted)
	opts->x_ix86_tune_string = NULL;

      /* If fpmath= is not set, and we now have sse2 on 32-bit, use it.  */
      if (enum_opts_set.x_ix86_fpmath)
	opts_set->x_ix86_fpmath = (enum fpmath_unit) 1;
      else if (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
	       && TARGET_SSE_P (opts->x_ix86_isa_flags))
	{
	  if (TARGET_80387_P (opts->x_target_flags))
	    opts->x_ix86_fpmath = (enum fpmath_unit) (FPMATH_SSE
						      | FPMATH_387);
	  else
	    opts->x_ix86_fpmath = (enum fpmath_unit) FPMATH_SSE;
	  opts_set->x_ix86_fpmath = (enum fpmath_unit) 1;
	}

      /* Do any overrides, such as arch=xxx, or tune=xxx support.  */
      bool r = ix86_option_override_internal (false, opts, opts_set);
      if (!r)
	{
	  release_options_strings (option_strings);
	  return error_mark_node;
	}

      /* Add any builtin functions with the new isa if any.  */
      ix86_add_new_builtins (opts->x_ix86_isa_flags, opts->x_ix86_isa_flags2);

      /* Save the current options unless we are validating options for
	 #pragma.  */
      t = build_target_option_node (opts);

      opts->x_ix86_arch_string = orig_arch_string;
      opts->x_ix86_tune_string = orig_tune_string;
      opts_set->x_ix86_fpmath = orig_fpmath_set;

      release_options_strings (option_strings);
    }

  return t;
}

/* Hook to validate attribute((target("string"))).  */

static bool
ix86_valid_target_attribute_p (tree fndecl,
			       tree ARG_UNUSED (name),
			       tree args,
			       int ARG_UNUSED (flags))
{
  struct gcc_options func_options;
  tree new_target, new_optimize;
  bool ret = true;

  /* attribute((target("default"))) does nothing, beyond
     affecting multi-versioning.  */
  if (TREE_VALUE (args)
      && TREE_CODE (TREE_VALUE (args)) == STRING_CST
      && TREE_CHAIN (args) == NULL_TREE
      && strcmp (TREE_STRING_POINTER (TREE_VALUE (args)), "default") == 0)
    return true;

  tree old_optimize = build_optimization_node (&global_options);

  /* Get the optimization options of the current function.  */  
  tree func_optimize = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl);
 
  if (!func_optimize)
    func_optimize = old_optimize;

  /* Init func_options.  */
  memset (&func_options, 0, sizeof (func_options));
  init_options_struct (&func_options, NULL);
  lang_hooks.init_options_struct (&func_options);
 
  cl_optimization_restore (&func_options,
			   TREE_OPTIMIZATION (func_optimize));

  /* Initialize func_options to the default before its target options can
     be set.  */
  cl_target_option_restore (&func_options,
			    TREE_TARGET_OPTION (target_option_default_node));

  new_target = ix86_valid_target_attribute_tree (args, &func_options,
						 &global_options_set);

  new_optimize = build_optimization_node (&func_options);

  if (new_target == error_mark_node)
    ret = false;

  else if (fndecl && new_target)
    {
      DECL_FUNCTION_SPECIFIC_TARGET (fndecl) = new_target;

      if (old_optimize != new_optimize)
	DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl) = new_optimize;
    }

  finalize_options_struct (&func_options);

  return ret;
}


/* Hook to determine if one function can safely inline another.  */

static bool
ix86_can_inline_p (tree caller, tree callee)
{
  bool ret = false;
  tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller);
  tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee);

  /* If callee has no option attributes, then it is ok to inline.  */
  if (!callee_tree)
    ret = true;

  /* If caller has no option attributes, but callee does then it is not ok to
     inline.  */
  else if (!caller_tree)
    ret = false;

  else
    {
      struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree);
      struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree);

      /* Callee's isa options should be a subset of the caller's, i.e. a SSE4
	 function can inline a SSE2 function but a SSE2 function can't inline
	 a SSE4 function.  */
      if (((caller_opts->x_ix86_isa_flags & callee_opts->x_ix86_isa_flags)
	   != callee_opts->x_ix86_isa_flags)
	  || ((caller_opts->x_ix86_isa_flags2 & callee_opts->x_ix86_isa_flags2)
	      != callee_opts->x_ix86_isa_flags2))
	ret = false;

      /* See if we have the same non-isa options.  */
      else if (caller_opts->x_target_flags != callee_opts->x_target_flags)
	ret = false;

      /* See if arch, tune, etc. are the same.  */
      else if (caller_opts->arch != callee_opts->arch)
	ret = false;

      else if (caller_opts->tune != callee_opts->tune)
	ret = false;

      else if (caller_opts->x_ix86_fpmath != callee_opts->x_ix86_fpmath)
	ret = false;

      else if (caller_opts->branch_cost != callee_opts->branch_cost)
	ret = false;

      else
	ret = true;
    }

  return ret;
}


/* Remember the last target of ix86_set_current_function.  */
static GTY(()) tree ix86_previous_fndecl;

/* Set targets globals to the default (or current #pragma GCC target
   if active).  Invalidate ix86_previous_fndecl cache.  */

void
ix86_reset_previous_fndecl (void)
{
  tree new_tree = target_option_current_node;
  cl_target_option_restore (&global_options, TREE_TARGET_OPTION (new_tree));
  if (TREE_TARGET_GLOBALS (new_tree))
    restore_target_globals (TREE_TARGET_GLOBALS (new_tree));
  else if (new_tree == target_option_default_node)
    restore_target_globals (&default_target_globals);
  else
    TREE_TARGET_GLOBALS (new_tree) = save_target_globals_default_opts ();
  ix86_previous_fndecl = NULL_TREE;
}

/* Set the func_type field from the function FNDECL.  */

static void
ix86_set_func_type (tree fndecl)
{
  if (cfun->machine->func_type == TYPE_UNKNOWN)
    {
      if (lookup_attribute ("interrupt",
			    TYPE_ATTRIBUTES (TREE_TYPE (fndecl))))
	{
	  int nargs = 0;
	  for (tree arg = DECL_ARGUMENTS (fndecl);
	       arg;
	       arg = TREE_CHAIN (arg))
	    nargs++;
	  cfun->machine->no_caller_saved_registers = true;
	  cfun->machine->func_type
	    = nargs == 2 ? TYPE_EXCEPTION : TYPE_INTERRUPT;

	  ix86_optimize_mode_switching[X86_DIRFLAG] = 1;

	  /* Only dwarf2out.c can handle -WORD(AP) as a pointer argument.  */
	  if (write_symbols != NO_DEBUG && write_symbols != DWARF2_DEBUG)
	    sorry ("Only DWARF debug format is supported for interrupt "
		   "service routine.");
	}
      else
	{
	  cfun->machine->func_type = TYPE_NORMAL;
	  if (lookup_attribute ("no_caller_saved_registers",
				TYPE_ATTRIBUTES (TREE_TYPE (fndecl))))
	    cfun->machine->no_caller_saved_registers = true;
	}
    }
}

/* Establish appropriate back-end context for processing the function
   FNDECL.  The argument might be NULL to indicate processing at top
   level, outside of any function scope.  */
static void
ix86_set_current_function (tree fndecl)
{
  /* Only change the context if the function changes.  This hook is called
     several times in the course of compiling a function, and we don't want to
     slow things down too much or call target_reinit when it isn't safe.  */
  if (fndecl == ix86_previous_fndecl)
    {
      /* There may be 2 function bodies for the same function FNDECL,
	 one is extern inline and one isn't.  Call ix86_set_func_type
	 to set the func_type field.  */
      if (fndecl != NULL_TREE)
	ix86_set_func_type (fndecl);
      return;
    }

  tree old_tree;
  if (ix86_previous_fndecl == NULL_TREE)
    old_tree = target_option_current_node;
  else if (DECL_FUNCTION_SPECIFIC_TARGET (ix86_previous_fndecl))
    old_tree = DECL_FUNCTION_SPECIFIC_TARGET (ix86_previous_fndecl);
  else
    old_tree = target_option_default_node;

  if (fndecl == NULL_TREE)
    {
      if (old_tree != target_option_current_node)
	ix86_reset_previous_fndecl ();
      return;
    }

  ix86_set_func_type (fndecl);

  tree new_tree = DECL_FUNCTION_SPECIFIC_TARGET (fndecl);
  if (new_tree == NULL_TREE)
    new_tree = target_option_default_node;

  if (old_tree != new_tree)
    {
      cl_target_option_restore (&global_options, TREE_TARGET_OPTION (new_tree));
      if (TREE_TARGET_GLOBALS (new_tree))
	restore_target_globals (TREE_TARGET_GLOBALS (new_tree));
      else if (new_tree == target_option_default_node)
	restore_target_globals (&default_target_globals);
      else
	TREE_TARGET_GLOBALS (new_tree) = save_target_globals_default_opts ();
    }
  ix86_previous_fndecl = fndecl;

  static bool prev_no_caller_saved_registers;

  /* 64-bit MS and SYSV ABI have different set of call used registers.
     Avoid expensive re-initialization of init_regs each time we switch
     function context.  */
  if (TARGET_64BIT
      && (call_used_regs[SI_REG]
	  == (cfun->machine->call_abi == MS_ABI)))
    reinit_regs ();
  /* Need to re-initialize init_regs if caller-saved registers are
     changed.  */
  else if (prev_no_caller_saved_registers
	   != cfun->machine->no_caller_saved_registers)
    reinit_regs ();

  if (cfun->machine->func_type != TYPE_NORMAL
      || cfun->machine->no_caller_saved_registers)
    {
      /* Don't allow MPX, SSE, MMX nor x87 instructions since they
	 may change processor state.  */
      const char *isa;
      if (TARGET_MPX)
	isa = "MPX";
      else if (TARGET_SSE)
	isa = "SSE";
      else if (TARGET_MMX)
	isa = "MMX/3Dnow";
      else if (TARGET_80387)
	isa = "80387";
      else
	isa = NULL;
      if (isa != NULL)
	{
	  if (cfun->machine->func_type != TYPE_NORMAL)
	    sorry ("%s instructions aren't allowed in %s service routine",
		   isa, (cfun->machine->func_type == TYPE_EXCEPTION
			 ? "exception" : "interrupt"));
	  else
	    sorry ("%s instructions aren't allowed in function with "
		   "no_caller_saved_registers attribute", isa);
	  /* Don't issue the same error twice.  */
	  cfun->machine->func_type = TYPE_NORMAL;
	  cfun->machine->no_caller_saved_registers = false;
	}
    }

  prev_no_caller_saved_registers
    = cfun->machine->no_caller_saved_registers;
}


/* Return true if this goes in large data/bss.  */

static bool
ix86_in_large_data_p (tree exp)
{
  if (ix86_cmodel != CM_MEDIUM && ix86_cmodel != CM_MEDIUM_PIC)
    return false;

  if (exp == NULL_TREE)
    return false;

  /* Functions are never large data.  */
  if (TREE_CODE (exp) == FUNCTION_DECL)
    return false;

  /* Automatic variables are never large data.  */
  if (VAR_P (exp) && !is_global_var (exp))
    return false;

  if (VAR_P (exp) && DECL_SECTION_NAME (exp))
    {
      const char *section = DECL_SECTION_NAME (exp);
      if (strcmp (section, ".ldata") == 0
	  || strcmp (section, ".lbss") == 0)
	return true;
      return false;
    }
  else
    {
      HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));

      /* If this is an incomplete type with size 0, then we can't put it
	 in data because it might be too big when completed.  Also,
	 int_size_in_bytes returns -1 if size can vary or is larger than
	 an integer in which case also it is safer to assume that it goes in
	 large data.  */
      if (size <= 0 || size > ix86_section_threshold)
	return true;
    }

  return false;
}

/* i386-specific section flag to mark large sections.  */
#define SECTION_LARGE SECTION_MACH_DEP

/* Switch to the appropriate section for output of DECL.
   DECL is either a `VAR_DECL' node or a constant of some sort.
   RELOC indicates whether forming the initial value of DECL requires
   link-time relocations.  */

ATTRIBUTE_UNUSED static section *
x86_64_elf_select_section (tree decl, int reloc,
			   unsigned HOST_WIDE_INT align)
{
  if (ix86_in_large_data_p (decl))
    {
      const char *sname = NULL;
      unsigned int flags = SECTION_WRITE | SECTION_LARGE;
      switch (categorize_decl_for_section (decl, reloc))
	{
	case SECCAT_DATA:
	  sname = ".ldata";
	  break;
	case SECCAT_DATA_REL:
	  sname = ".ldata.rel";
	  break;
	case SECCAT_DATA_REL_LOCAL:
	  sname = ".ldata.rel.local";
	  break;
	case SECCAT_DATA_REL_RO:
	  sname = ".ldata.rel.ro";
	  break;
	case SECCAT_DATA_REL_RO_LOCAL:
	  sname = ".ldata.rel.ro.local";
	  break;
	case SECCAT_BSS:
	  sname = ".lbss";
	  flags |= SECTION_BSS;
	  break;
	case SECCAT_RODATA:
	case SECCAT_RODATA_MERGE_STR:
	case SECCAT_RODATA_MERGE_STR_INIT:
	case SECCAT_RODATA_MERGE_CONST:
	  sname = ".lrodata";
	  flags &= ~SECTION_WRITE;
	  break;
	case SECCAT_SRODATA:
	case SECCAT_SDATA:
	case SECCAT_SBSS:
	  gcc_unreachable ();
	case SECCAT_TEXT:
	case SECCAT_TDATA:
	case SECCAT_TBSS:
	  /* We don't split these for medium model.  Place them into
	     default sections and hope for best.  */
	  break;
	}
      if (sname)
	{
	  /* We might get called with string constants, but get_named_section
	     doesn't like them as they are not DECLs.  Also, we need to set
	     flags in that case.  */
	  if (!DECL_P (decl))
	    return get_section (sname, flags, NULL);
	  return get_named_section (decl, sname, reloc);
	}
    }
  return default_elf_select_section (decl, reloc, align);
}

/* Select a set of attributes for section NAME based on the properties
   of DECL and whether or not RELOC indicates that DECL's initializer
   might contain runtime relocations.  */

static unsigned int ATTRIBUTE_UNUSED
x86_64_elf_section_type_flags (tree decl, const char *name, int reloc)
{
  unsigned int flags = default_section_type_flags (decl, name, reloc);

  if (ix86_in_large_data_p (decl))
    flags |= SECTION_LARGE;

  if (decl == NULL_TREE
      && (strcmp (name, ".ldata.rel.ro") == 0
	  || strcmp (name, ".ldata.rel.ro.local") == 0))
    flags |= SECTION_RELRO;

  if (strcmp (name, ".lbss") == 0
      || strncmp (name, ".lbss.", 5) == 0
      || strncmp (name, ".gnu.linkonce.lb.", 16) == 0)
    flags |= SECTION_BSS;

  return flags;
}

/* Build up a unique section name, expressed as a
   STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
   RELOC indicates whether the initial value of EXP requires
   link-time relocations.  */

static void ATTRIBUTE_UNUSED
x86_64_elf_unique_section (tree decl, int reloc)
{
  if (ix86_in_large_data_p (decl))
    {
      const char *prefix = NULL;
      /* We only need to use .gnu.linkonce if we don't have COMDAT groups.  */
      bool one_only = DECL_COMDAT_GROUP (decl) && !HAVE_COMDAT_GROUP;

      switch (categorize_decl_for_section (decl, reloc))
	{
	case SECCAT_DATA:
	case SECCAT_DATA_REL:
	case SECCAT_DATA_REL_LOCAL:
	case SECCAT_DATA_REL_RO:
	case SECCAT_DATA_REL_RO_LOCAL:
          prefix = one_only ? ".ld" : ".ldata";
	  break;
	case SECCAT_BSS:
          prefix = one_only ? ".lb" : ".lbss";
	  break;
	case SECCAT_RODATA:
	case SECCAT_RODATA_MERGE_STR:
	case SECCAT_RODATA_MERGE_STR_INIT:
	case SECCAT_RODATA_MERGE_CONST:
          prefix = one_only ? ".lr" : ".lrodata";
	  break;
	case SECCAT_SRODATA:
	case SECCAT_SDATA:
	case SECCAT_SBSS:
	  gcc_unreachable ();
	case SECCAT_TEXT:
	case SECCAT_TDATA:
	case SECCAT_TBSS:
	  /* We don't split these for medium model.  Place them into
	     default sections and hope for best.  */
	  break;
	}
      if (prefix)
	{
	  const char *name, *linkonce;
	  char *string;

	  name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
	  name = targetm.strip_name_encoding (name);

	  /* If we're using one_only, then there needs to be a .gnu.linkonce
     	     prefix to the section name.  */
	  linkonce = one_only ? ".gnu.linkonce" : "";

	  string = ACONCAT ((linkonce, prefix, ".", name, NULL));

	  set_decl_section_name (decl, string);
	  return;
	}
    }
  default_unique_section (decl, reloc);
}

#ifdef COMMON_ASM_OP

#ifndef LARGECOMM_SECTION_ASM_OP
#define LARGECOMM_SECTION_ASM_OP "\t.largecomm\t"
#endif

/* This says how to output assembler code to declare an
   uninitialized external linkage data object.

   For medium model x86-64 we need to use LARGECOMM_SECTION_ASM_OP opcode for
   large objects.  */
void
x86_elf_aligned_decl_common (FILE *file, tree decl,
			const char *name, unsigned HOST_WIDE_INT size,
			int align)
{
  if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
      && size > (unsigned int)ix86_section_threshold)
    {
      switch_to_section (get_named_section (decl, ".lbss", 0));
      fputs (LARGECOMM_SECTION_ASM_OP, file);
    }
  else
    fputs (COMMON_ASM_OP, file);
  assemble_name (file, name);
  fprintf (file, "," HOST_WIDE_INT_PRINT_UNSIGNED ",%u\n",
	   size, align / BITS_PER_UNIT);
}
#endif

/* Utility function for targets to use in implementing
   ASM_OUTPUT_ALIGNED_BSS.  */

void
x86_output_aligned_bss (FILE *file, tree decl, const char *name,
		       	unsigned HOST_WIDE_INT size, int align)
{
  if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
      && size > (unsigned int)ix86_section_threshold)
    switch_to_section (get_named_section (decl, ".lbss", 0));
  else
    switch_to_section (bss_section);
  ASM_OUTPUT_ALIGN (file, floor_log2 (align / BITS_PER_UNIT));
#ifdef ASM_DECLARE_OBJECT_NAME
  last_assemble_variable_decl = decl;
  ASM_DECLARE_OBJECT_NAME (file, name, decl);
#else
  /* Standard thing is just output label for the object.  */
  ASM_OUTPUT_LABEL (file, name);
#endif /* ASM_DECLARE_OBJECT_NAME */
  ASM_OUTPUT_SKIP (file, size ? size : 1);
}

/* Decide whether we must probe the stack before any space allocation
   on this target.  It's essentially TARGET_STACK_PROBE except when
   -fstack-check causes the stack to be already probed differently.  */

bool
ix86_target_stack_probe (void)
{
  /* Do not probe the stack twice if static stack checking is enabled.  */
  if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
    return false;

  return TARGET_STACK_PROBE;
}

/* Decide whether we can make a sibling call to a function.  DECL is the
   declaration of the function being targeted by the call and EXP is the
   CALL_EXPR representing the call.  */

static bool
ix86_function_ok_for_sibcall (tree decl, tree exp)
{
  tree type, decl_or_type;
  rtx a, b;
  bool bind_global = decl && !targetm.binds_local_p (decl);

  /* Sibling call isn't OK if there are no caller-saved registers
     since all registers must be preserved before return.  */
  if (cfun->machine->no_caller_saved_registers)
    return false;

  /* If we are generating position-independent code, we cannot sibcall
     optimize direct calls to global functions, as the PLT requires
     %ebx be live. (Darwin does not have a PLT.)  */
  if (!TARGET_MACHO
      && !TARGET_64BIT
      && flag_pic
      && flag_plt
      && bind_global)
    return false;

  /* If we need to align the outgoing stack, then sibcalling would
     unalign the stack, which may break the called function.  */
  if (ix86_minimum_incoming_stack_boundary (true)
      < PREFERRED_STACK_BOUNDARY)
    return false;

  if (decl)
    {
      decl_or_type = decl;
      type = TREE_TYPE (decl);
    }
  else
    {
      /* We're looking at the CALL_EXPR, we need the type of the function.  */
      type = CALL_EXPR_FN (exp);		/* pointer expression */
      type = TREE_TYPE (type);			/* pointer type */
      type = TREE_TYPE (type);			/* function type */
      decl_or_type = type;
    }

  /* Check that the return value locations are the same.  Like
     if we are returning floats on the 80387 register stack, we cannot
     make a sibcall from a function that doesn't return a float to a
     function that does or, conversely, from a function that does return
     a float to a function that doesn't; the necessary stack adjustment
     would not be executed.  This is also the place we notice
     differences in the return value ABI.  Note that it is ok for one
     of the functions to have void return type as long as the return
     value of the other is passed in a register.  */
  a = ix86_function_value (TREE_TYPE (exp), decl_or_type, false);
  b = ix86_function_value (TREE_TYPE (DECL_RESULT (cfun->decl)),
			   cfun->decl, false);
  if (STACK_REG_P (a) || STACK_REG_P (b))
    {
      if (!rtx_equal_p (a, b))
	return false;
    }
  else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
    ;
  else if (!rtx_equal_p (a, b))
    return false;

  if (TARGET_64BIT)
    {
      /* The SYSV ABI has more call-clobbered registers;
	 disallow sibcalls from MS to SYSV.  */
      if (cfun->machine->call_abi == MS_ABI
	  && ix86_function_type_abi (type) == SYSV_ABI)
	return false;
    }
  else
    {
      /* If this call is indirect, we'll need to be able to use a
	 call-clobbered register for the address of the target function.
	 Make sure that all such registers are not used for passing
	 parameters.  Note that DLLIMPORT functions and call to global
	 function via GOT slot are indirect.  */
      if (!decl
	  || (bind_global && flag_pic && !flag_plt)
	  || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && DECL_DLLIMPORT_P (decl)))
	{
	  /* Check if regparm >= 3 since arg_reg_available is set to
	     false if regparm == 0.  If regparm is 1 or 2, there is
	     always a call-clobbered register available.

	     ??? The symbol indirect call doesn't need a call-clobbered
	     register.  But we don't know if this is a symbol indirect
	     call or not here.  */
	  if (ix86_function_regparm (type, NULL) >= 3
	      && !cfun->machine->arg_reg_available)
	    return false;
	}
    }

  /* Otherwise okay.  That also includes certain types of indirect calls.  */
  return true;
}

/* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
   and "sseregparm" calling convention attributes;
   arguments as in struct attribute_spec.handler.  */

static tree
ix86_handle_cconv_attribute (tree *node, tree name,
				   tree args,
				   int,
				   bool *no_add_attrs)
{
  if (TREE_CODE (*node) != FUNCTION_TYPE
      && TREE_CODE (*node) != METHOD_TYPE
      && TREE_CODE (*node) != FIELD_DECL
      && TREE_CODE (*node) != TYPE_DECL)
    {
      warning (OPT_Wattributes, "%qE attribute only applies to functions",
	       name);
      *no_add_attrs = true;
      return NULL_TREE;
    }

  /* Can combine regparm with all attributes but fastcall, and thiscall.  */
  if (is_attribute_p ("regparm", name))
    {
      tree cst;

      if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
        {
	  error ("fastcall and regparm attributes are not compatible");
	}

      if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
	{
	  error ("regparam and thiscall attributes are not compatible");
	}

      cst = TREE_VALUE (args);
      if (TREE_CODE (cst) != INTEGER_CST)
	{
	  warning (OPT_Wattributes,
		   "%qE attribute requires an integer constant argument",
		   name);
	  *no_add_attrs = true;
	}
      else if (compare_tree_int (cst, REGPARM_MAX) > 0)
	{
	  warning (OPT_Wattributes, "argument to %qE attribute larger than %d",
		   name, REGPARM_MAX);
	  *no_add_attrs = true;
	}

      return NULL_TREE;
    }

  if (TARGET_64BIT)
    {
      /* Do not warn when emulating the MS ABI.  */
      if ((TREE_CODE (*node) != FUNCTION_TYPE
	   && TREE_CODE (*node) != METHOD_TYPE)
	  || ix86_function_type_abi (*node) != MS_ABI)
	warning (OPT_Wattributes, "%qE attribute ignored",
	         name);
      *no_add_attrs = true;
      return NULL_TREE;
    }

  /* Can combine fastcall with stdcall (redundant) and sseregparm.  */
  if (is_attribute_p ("fastcall", name))
    {
      if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
        {
	  error ("fastcall and cdecl attributes are not compatible");
	}
      if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
        {
	  error ("fastcall and stdcall attributes are not compatible");
	}
      if (lookup_attribute ("regparm", TYPE_ATTRIBUTES (*node)))
        {
	  error ("fastcall and regparm attributes are not compatible");
	}
      if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
	{
	  error ("fastcall and thiscall attributes are not compatible");
	}
    }

  /* Can combine stdcall with fastcall (redundant), regparm and
     sseregparm.  */
  else if (is_attribute_p ("stdcall", name))
    {
      if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
        {
	  error ("stdcall and cdecl attributes are not compatible");
	}
      if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
        {
	  error ("stdcall and fastcall attributes are not compatible");
	}
      if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
	{
	  error ("stdcall and thiscall attributes are not compatible");
	}
    }

  /* Can combine cdecl with regparm and sseregparm.  */
  else if (is_attribute_p ("cdecl", name))
    {
      if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
        {
	  error ("stdcall and cdecl attributes are not compatible");
	}
      if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
        {
	  error ("fastcall and cdecl attributes are not compatible");
	}
      if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
	{
	  error ("cdecl and thiscall attributes are not compatible");
	}
    }
  else if (is_attribute_p ("thiscall", name))
    {
      if (TREE_CODE (*node) != METHOD_TYPE && pedantic)
	warning (OPT_Wattributes, "%qE attribute is used for non-class method",
	         name);
      if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
	{
	  error ("stdcall and thiscall attributes are not compatible");
	}
      if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
	{
	  error ("fastcall and thiscall attributes are not compatible");
	}
      if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
	{
	  error ("cdecl and thiscall attributes are not compatible");
	}
    }

  /* Can combine sseregparm with all attributes.  */

  return NULL_TREE;
}

/* The transactional memory builtins are implicitly regparm or fastcall
   depending on the ABI.  Override the generic do-nothing attribute that
   these builtins were declared with, and replace it with one of the two
   attributes that we expect elsewhere.  */

static tree
ix86_handle_tm_regparm_attribute (tree *node, tree, tree,
				  int flags, bool *no_add_attrs)
{
  tree alt;

  /* In no case do we want to add the placeholder attribute.  */
  *no_add_attrs = true;

  /* The 64-bit ABI is unchanged for transactional memory.  */
  if (TARGET_64BIT)
    return NULL_TREE;

  /* ??? Is there a better way to validate 32-bit windows?  We have
     cfun->machine->call_abi, but that seems to be set only for 64-bit.  */
  if (CHECK_STACK_LIMIT > 0)
    alt = tree_cons (get_identifier ("fastcall"), NULL, NULL);
  else
    {
      alt = tree_cons (NULL, build_int_cst (NULL, 2), NULL);
      alt = tree_cons (get_identifier ("regparm"), alt, NULL);
    }
  decl_attributes (node, alt, flags);

  return NULL_TREE;
}

/* This function determines from TYPE the calling-convention.  */

unsigned int
ix86_get_callcvt (const_tree type)
{
  unsigned int ret = 0;
  bool is_stdarg;
  tree attrs;

  if (TARGET_64BIT)
    return IX86_CALLCVT_CDECL;

  attrs = TYPE_ATTRIBUTES (type);
  if (attrs != NULL_TREE)
    {
      if (lookup_attribute ("cdecl", attrs))
	ret |= IX86_CALLCVT_CDECL;
      else if (lookup_attribute ("stdcall", attrs))
	ret |= IX86_CALLCVT_STDCALL;
      else if (lookup_attribute ("fastcall", attrs))
	ret |= IX86_CALLCVT_FASTCALL;
      else if (lookup_attribute ("thiscall", attrs))
	ret |= IX86_CALLCVT_THISCALL;

      /* Regparam isn't allowed for thiscall and fastcall.  */
      if ((ret & (IX86_CALLCVT_THISCALL | IX86_CALLCVT_FASTCALL)) == 0)
	{
	  if (lookup_attribute ("regparm", attrs))
	    ret |= IX86_CALLCVT_REGPARM;
	  if (lookup_attribute ("sseregparm", attrs))
	    ret |= IX86_CALLCVT_SSEREGPARM;
	}

      if (IX86_BASE_CALLCVT(ret) != 0)
	return ret;
    }

  is_stdarg = stdarg_p (type);
  if (TARGET_RTD && !is_stdarg)
    return IX86_CALLCVT_STDCALL | ret;

  if (ret != 0
      || is_stdarg
      || TREE_CODE (type) != METHOD_TYPE
      || ix86_function_type_abi (type) != MS_ABI)
    return IX86_CALLCVT_CDECL | ret;

  return IX86_CALLCVT_THISCALL;
}

/* Return 0 if the attributes for two types are incompatible, 1 if they
   are compatible, and 2 if they are nearly compatible (which causes a
   warning to be generated).  */

static int
ix86_comp_type_attributes (const_tree type1, const_tree type2)
{
  unsigned int ccvt1, ccvt2;

  if (TREE_CODE (type1) != FUNCTION_TYPE
      && TREE_CODE (type1) != METHOD_TYPE)
    return 1;

  ccvt1 = ix86_get_callcvt (type1);
  ccvt2 = ix86_get_callcvt (type2);
  if (ccvt1 != ccvt2)
    return 0;
  if (ix86_function_regparm (type1, NULL)
      != ix86_function_regparm (type2, NULL))
    return 0;

  return 1;
}

/* Return the regparm value for a function with the indicated TYPE and DECL.
   DECL may be NULL when calling function indirectly
   or considering a libcall.  */

static int
ix86_function_regparm (const_tree type, const_tree decl)
{
  tree attr;
  int regparm;
  unsigned int ccvt;

  if (TARGET_64BIT)
    return (ix86_function_type_abi (type) == SYSV_ABI
	    ? X86_64_REGPARM_MAX : X86_64_MS_REGPARM_MAX);
  ccvt = ix86_get_callcvt (type);
  regparm = ix86_regparm;

  if ((ccvt & IX86_CALLCVT_REGPARM) != 0)
    {
      attr = lookup_attribute ("regparm", TYPE_ATTRIBUTES (type));
      if (attr)
	{
	  regparm = TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr)));
	  return regparm;
	}
    }
  else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
    return 2;
  else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
    return 1;

  /* Use register calling convention for local functions when possible.  */
  if (decl
      && TREE_CODE (decl) == FUNCTION_DECL)
    {
      cgraph_node *target = cgraph_node::get (decl);
      if (target)
	target = target->function_symbol ();

      /* Caller and callee must agree on the calling convention, so
	 checking here just optimize means that with
	 __attribute__((optimize (...))) caller could use regparm convention
	 and callee not, or vice versa.  Instead look at whether the callee
	 is optimized or not.  */
      if (target && opt_for_fn (target->decl, optimize)
	  && !(profile_flag && !flag_fentry))
	{
	  cgraph_local_info *i = &target->local;
	  if (i && i->local && i->can_change_signature)
	    {
	      int local_regparm, globals = 0, regno;

	      /* Make sure no regparm register is taken by a
		 fixed register variable.  */
	      for (local_regparm = 0; local_regparm < REGPARM_MAX;
		   local_regparm++)
		if (fixed_regs[local_regparm])
		  break;

	      /* We don't want to use regparm(3) for nested functions as
		 these use a static chain pointer in the third argument.  */
	      if (local_regparm == 3 && DECL_STATIC_CHAIN (target->decl))
		local_regparm = 2;

	      /* Save a register for the split stack.  */
	      if (flag_split_stack)
		{
		  if (local_regparm == 3)
		    local_regparm = 2;
		  else if (local_regparm == 2
			   && DECL_STATIC_CHAIN (target->decl))
		    local_regparm = 1;
		}

	      /* Each fixed register usage increases register pressure,
		 so less registers should be used for argument passing.
		 This functionality can be overriden by an explicit
		 regparm value.  */
	      for (regno = AX_REG; regno <= DI_REG; regno++)
		if (fixed_regs[regno])
		  globals++;

	      local_regparm
		= globals < local_regparm ? local_regparm - globals : 0;

	      if (local_regparm > regparm)
		regparm = local_regparm;
	    }
	}
    }

  return regparm;
}

/* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
   DFmode (2) arguments in SSE registers for a function with the
   indicated TYPE and DECL.  DECL may be NULL when calling function
   indirectly or considering a libcall.  Return -1 if any FP parameter
   should be rejected by error.  This is used in siutation we imply SSE
   calling convetion but the function is called from another function with
   SSE disabled. Otherwise return 0.  */

static int
ix86_function_sseregparm (const_tree type, const_tree decl, bool warn)
{
  gcc_assert (!TARGET_64BIT);

  /* Use SSE registers to pass SFmode and DFmode arguments if requested
     by the sseregparm attribute.  */
  if (TARGET_SSEREGPARM
      || (type && lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type))))
    {
      if (!TARGET_SSE)
	{
	  if (warn)
	    {
	      if (decl)
		error ("calling %qD with attribute sseregparm without "
		       "SSE/SSE2 enabled", decl);
	      else
		error ("calling %qT with attribute sseregparm without "
		       "SSE/SSE2 enabled", type);
	    }
	  return 0;
	}

      return 2;
    }

  if (!decl)
    return 0;

  cgraph_node *target = cgraph_node::get (decl);
  if (target)
    target = target->function_symbol ();

  /* For local functions, pass up to SSE_REGPARM_MAX SFmode
     (and DFmode for SSE2) arguments in SSE registers.  */
  if (target
      /* TARGET_SSE_MATH */
      && (target_opts_for_fn (target->decl)->x_ix86_fpmath & FPMATH_SSE)
      && opt_for_fn (target->decl, optimize)
      && !(profile_flag && !flag_fentry))
    {
      cgraph_local_info *i = &target->local;
      if (i && i->local && i->can_change_signature)
	{
	  /* Refuse to produce wrong code when local function with SSE enabled
	     is called from SSE disabled function.
	     FIXME: We need a way to detect these cases cross-ltrans partition
	     and avoid using SSE calling conventions on local functions called
	     from function with SSE disabled.  For now at least delay the
	     warning until we know we are going to produce wrong code.
	     See PR66047  */
	  if (!TARGET_SSE && warn)
	    return -1;
	  return TARGET_SSE2_P (target_opts_for_fn (target->decl)
				->x_ix86_isa_flags) ? 2 : 1;
	}
    }

  return 0;
}

/* Return true if EAX is live at the start of the function.  Used by
   ix86_expand_prologue to determine if we need special help before
   calling allocate_stack_worker.  */

static bool
ix86_eax_live_at_start_p (void)
{
  /* Cheat.  Don't bother working forward from ix86_function_regparm
     to the function type to whether an actual argument is located in
     eax.  Instead just look at cfg info, which is still close enough
     to correct at this point.  This gives false positives for broken
     functions that might use uninitialized data that happens to be
     allocated in eax, but who cares?  */
  return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 0);
}

static bool
ix86_keep_aggregate_return_pointer (tree fntype)
{
  tree attr;

  if (!TARGET_64BIT)
    {
      attr = lookup_attribute ("callee_pop_aggregate_return",
			       TYPE_ATTRIBUTES (fntype));
      if (attr)
	return (TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr))) == 0);

      /* For 32-bit MS-ABI the default is to keep aggregate
         return pointer.  */
      if (ix86_function_type_abi (fntype) == MS_ABI)
	return true;
    }
  return KEEP_AGGREGATE_RETURN_POINTER != 0;
}

/* Value is the number of bytes of arguments automatically
   popped when returning from a subroutine call.
   FUNDECL is the declaration node of the function (as a tree),
   FUNTYPE is the data type of the function (as a tree),
   or for a library call it is an identifier node for the subroutine name.
   SIZE is the number of bytes of arguments passed on the stack.

   On the 80386, the RTD insn may be used to pop them if the number
     of args is fixed, but if the number is variable then the caller
     must pop them all.  RTD can't be used for library calls now
     because the library is compiled with the Unix compiler.
   Use of RTD is a selectable option, since it is incompatible with
   standard Unix calling sequences.  If the option is not selected,
   the caller must always pop the args.

   The attribute stdcall is equivalent to RTD on a per module basis.  */

static int
ix86_return_pops_args (tree fundecl, tree funtype, int size)
{
  unsigned int ccvt;

  /* None of the 64-bit ABIs pop arguments.  */
  if (TARGET_64BIT)
    return 0;

  ccvt = ix86_get_callcvt (funtype);

  if ((ccvt & (IX86_CALLCVT_STDCALL | IX86_CALLCVT_FASTCALL
	       | IX86_CALLCVT_THISCALL)) != 0
      && ! stdarg_p (funtype))
    return size;

  /* Lose any fake structure return argument if it is passed on the stack.  */
  if (aggregate_value_p (TREE_TYPE (funtype), fundecl)
      && !ix86_keep_aggregate_return_pointer (funtype))
    {
      int nregs = ix86_function_regparm (funtype, fundecl);
      if (nregs == 0)
	return GET_MODE_SIZE (Pmode);
    }

  return 0;
}

/* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook.  */

static bool
ix86_legitimate_combined_insn (rtx_insn *insn)
{
  int i;

  /* Check operand constraints in case hard registers were propagated
     into insn pattern.  This check prevents combine pass from
     generating insn patterns with invalid hard register operands.
     These invalid insns can eventually confuse reload to error out
     with a spill failure.  See also PRs 46829 and 46843.  */

  gcc_assert (INSN_CODE (insn) >= 0);

  extract_insn (insn);
  preprocess_constraints (insn);

  int n_operands = recog_data.n_operands;
  int n_alternatives = recog_data.n_alternatives;
  for (i = 0; i < n_operands; i++)
    {
      rtx op = recog_data.operand[i];
      machine_mode mode = GET_MODE (op);
      const operand_alternative *op_alt;
      int offset = 0;
      bool win;
      int j;

      /* A unary operator may be accepted by the predicate, but it
	 is irrelevant for matching constraints.  */
      if (UNARY_P (op))
	op = XEXP (op, 0);

      if (SUBREG_P (op))
	{
	  if (REG_P (SUBREG_REG (op))
	      && REGNO (SUBREG_REG (op)) < FIRST_PSEUDO_REGISTER)
	    offset = subreg_regno_offset (REGNO (SUBREG_REG (op)),
					  GET_MODE (SUBREG_REG (op)),
					  SUBREG_BYTE (op),
					  GET_MODE (op));
	  op = SUBREG_REG (op);
	}

      if (!(REG_P (op) && HARD_REGISTER_P (op)))
	continue;

      op_alt = recog_op_alt;

      /* Operand has no constraints, anything is OK.  */
      win = !n_alternatives;

      alternative_mask preferred = get_preferred_alternatives (insn);
      for (j = 0; j < n_alternatives; j++, op_alt += n_operands)
	{
	  if (!TEST_BIT (preferred, j))
	    continue;
	  if (op_alt[i].anything_ok
	      || (op_alt[i].matches != -1
		  && operands_match_p
		  (recog_data.operand[i],
		   recog_data.operand[op_alt[i].matches]))
	      || reg_fits_class_p (op, op_alt[i].cl, offset, mode))
	    {
	      win = true;
	      break;
	    }
	}

      if (!win)
	return false;
    }

  return true;
}

/* Implement the TARGET_ASAN_SHADOW_OFFSET hook.  */

static unsigned HOST_WIDE_INT
ix86_asan_shadow_offset (void)
{
  return TARGET_LP64 ? (TARGET_MACHO ? (HOST_WIDE_INT_1 << 44)
				     : HOST_WIDE_INT_C (0x7fff8000))
		     : (HOST_WIDE_INT_1 << 29);
}

/* Argument support functions.  */

/* Return true when register may be used to pass function parameters.  */
bool
ix86_function_arg_regno_p (int regno)
{
  int i;
  enum calling_abi call_abi;
  const int *parm_regs;

  if (TARGET_MPX && BND_REGNO_P (regno))
    return true;

  if (!TARGET_64BIT)
    {
      if (TARGET_MACHO)
        return (regno < REGPARM_MAX
                || (TARGET_SSE && SSE_REGNO_P (regno) && !fixed_regs[regno]));
      else
        return (regno < REGPARM_MAX
	        || (TARGET_MMX && MMX_REGNO_P (regno)
	  	    && (regno < FIRST_MMX_REG + MMX_REGPARM_MAX))
	        || (TARGET_SSE && SSE_REGNO_P (regno)
		    && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX)));
    }

  if (TARGET_SSE && SSE_REGNO_P (regno)
      && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX))
    return true;

  /* TODO: The function should depend on current function ABI but
     builtins.c would need updating then. Therefore we use the
     default ABI.  */
  call_abi = ix86_cfun_abi ();

  /* RAX is used as hidden argument to va_arg functions.  */
  if (call_abi == SYSV_ABI && regno == AX_REG)
    return true;

  if (call_abi == MS_ABI)
    parm_regs = x86_64_ms_abi_int_parameter_registers;
  else
    parm_regs = x86_64_int_parameter_registers;

  for (i = 0; i < (call_abi == MS_ABI
		   ? X86_64_MS_REGPARM_MAX : X86_64_REGPARM_MAX); i++)
    if (regno == parm_regs[i])
      return true;
  return false;
}

/* Return if we do not know how to pass TYPE solely in registers.  */

static bool
ix86_must_pass_in_stack (machine_mode mode, const_tree type)
{
  if (must_pass_in_stack_var_size_or_pad (mode, type))
    return true;

  /* For 32-bit, we want TImode aggregates to go on the stack.  But watch out!
     The layout_type routine is crafty and tries to trick us into passing
     currently unsupported vector types on the stack by using TImode.  */
  return (!TARGET_64BIT && mode == TImode
	  && type && TREE_CODE (type) != VECTOR_TYPE);
}

/* It returns the size, in bytes, of the area reserved for arguments passed
   in registers for the function represented by fndecl dependent to the used
   abi format.  */
int
ix86_reg_parm_stack_space (const_tree fndecl)
{
  enum calling_abi call_abi = SYSV_ABI;
  if (fndecl != NULL_TREE && TREE_CODE (fndecl) == FUNCTION_DECL)
    call_abi = ix86_function_abi (fndecl);
  else
    call_abi = ix86_function_type_abi (fndecl);
  if (TARGET_64BIT && call_abi == MS_ABI)
    return 32;
  return 0;
}

/* We add this as a workaround in order to use libc_has_function
   hook in i386.md.  */
bool
ix86_libc_has_function (enum function_class fn_class)
{
  return targetm.libc_has_function (fn_class);
}

/* Returns value SYSV_ABI, MS_ABI dependent on fntype,
   specifying the call abi used.  */
enum calling_abi
ix86_function_type_abi (const_tree fntype)
{
  enum calling_abi abi = ix86_abi;

  if (fntype == NULL_TREE || TYPE_ATTRIBUTES (fntype) == NULL_TREE)
    return abi;

  if (abi == SYSV_ABI
      && lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (fntype)))
    {
      if (TARGET_X32)
	error ("X32 does not support ms_abi attribute");

      abi = MS_ABI;
    }
  else if (abi == MS_ABI
	   && lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (fntype)))
    abi = SYSV_ABI;

  return abi;
}

static enum calling_abi
ix86_function_abi (const_tree fndecl)
{
  return fndecl ? ix86_function_type_abi (TREE_TYPE (fndecl)) : ix86_abi;
}

/* Returns value SYSV_ABI, MS_ABI dependent on cfun,
   specifying the call abi used.  */
enum calling_abi
ix86_cfun_abi (void)
{
  return cfun ? cfun->machine->call_abi : ix86_abi;
}

static bool
ix86_function_ms_hook_prologue (const_tree fn)
{
  if (fn && lookup_attribute ("ms_hook_prologue", DECL_ATTRIBUTES (fn)))
    {
      if (decl_function_context (fn) != NULL_TREE)
	error_at (DECL_SOURCE_LOCATION (fn),
		  "ms_hook_prologue is not compatible with nested function");
      else
        return true;
    }
  return false;
}

/* Write the extra assembler code needed to declare a function properly.  */

void
ix86_asm_output_function_label (FILE *asm_out_file, const char *fname,
				tree decl)
{
  bool is_ms_hook = ix86_function_ms_hook_prologue (decl);

  if (is_ms_hook)
    {
      int i, filler_count = (TARGET_64BIT ? 32 : 16);
      unsigned int filler_cc = 0xcccccccc;

      for (i = 0; i < filler_count; i += 4)
        fprintf (asm_out_file, ASM_LONG " %#x\n", filler_cc);
    }

#ifdef SUBTARGET_ASM_UNWIND_INIT
  SUBTARGET_ASM_UNWIND_INIT (asm_out_file);
#endif

  ASM_OUTPUT_LABEL (asm_out_file, fname);

  /* Output magic byte marker, if hot-patch attribute is set.  */
  if (is_ms_hook)
    {
      if (TARGET_64BIT)
	{
	  /* leaq [%rsp + 0], %rsp  */
	  asm_fprintf (asm_out_file, ASM_BYTE
		       "0x48, 0x8d, 0xa4, 0x24, 0x00, 0x00, 0x00, 0x00\n");
	}
      else
	{
          /* movl.s %edi, %edi
	     push   %ebp
	     movl.s %esp, %ebp */
	  asm_fprintf (asm_out_file, ASM_BYTE
		       "0x8b, 0xff, 0x55, 0x8b, 0xec\n");
	}
    }
}

/* regclass.c  */
extern void init_regs (void);

/* Implementation of call abi switching target hook. Specific to FNDECL
   the specific call register sets are set.  See also
   ix86_conditional_register_usage for more details.  */
void
ix86_call_abi_override (const_tree fndecl)
{
  cfun->machine->call_abi = ix86_function_abi (fndecl);
}

/* Return 1 if pseudo register should be created and used to hold
   GOT address for PIC code.  */
bool
ix86_use_pseudo_pic_reg (void)
{
  if ((TARGET_64BIT
       && (ix86_cmodel == CM_SMALL_PIC
	   || TARGET_PECOFF))
      || !flag_pic)
    return false;
  return true;
}

/* Initialize large model PIC register.  */

static void
ix86_init_large_pic_reg (unsigned int tmp_regno)
{
  rtx_code_label *label;
  rtx tmp_reg;

  gcc_assert (Pmode == DImode);
  label = gen_label_rtx ();
  emit_label (label);
  LABEL_PRESERVE_P (label) = 1;
  tmp_reg = gen_rtx_REG (Pmode, tmp_regno);
  gcc_assert (REGNO (pic_offset_table_rtx) != tmp_regno);
  emit_insn (gen_set_rip_rex64 (pic_offset_table_rtx,
				label));
  emit_insn (gen_set_got_offset_rex64 (tmp_reg, label));
  emit_insn (ix86_gen_add3 (pic_offset_table_rtx,
			    pic_offset_table_rtx, tmp_reg));
}

/* Create and initialize PIC register if required.  */
static void
ix86_init_pic_reg (void)
{
  edge entry_edge;
  rtx_insn *seq;

  if (!ix86_use_pseudo_pic_reg ())
    return;

  start_sequence ();

  if (TARGET_64BIT)
    {
      if (ix86_cmodel == CM_LARGE_PIC)
	ix86_init_large_pic_reg (R11_REG);
      else
	emit_insn (gen_set_got_rex64 (pic_offset_table_rtx));
    }
  else
    {
      /*  If there is future mcount call in the function it is more profitable
	  to emit SET_GOT into ABI defined REAL_PIC_OFFSET_TABLE_REGNUM.  */
      rtx reg = crtl->profile
		? gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM)
		: pic_offset_table_rtx;
      rtx_insn *insn = emit_insn (gen_set_got (reg));
      RTX_FRAME_RELATED_P (insn) = 1;
      if (crtl->profile)
        emit_move_insn (pic_offset_table_rtx, reg);
      add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
    }

  seq = get_insns ();
  end_sequence ();

  entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
  insert_insn_on_edge (seq, entry_edge);
  commit_one_edge_insertion (entry_edge);
}

/* Initialize a variable CUM of type CUMULATIVE_ARGS
   for a call to a function whose data type is FNTYPE.
   For a library call, FNTYPE is 0.  */

void
init_cumulative_args (CUMULATIVE_ARGS *cum,  /* Argument info to initialize */
		      tree fntype,	/* tree ptr for function decl */
		      rtx libname,	/* SYMBOL_REF of library name or 0 */
		      tree fndecl,
		      int caller)
{
  struct cgraph_local_info *i = NULL;
  struct cgraph_node *target = NULL;

  memset (cum, 0, sizeof (*cum));

  if (fndecl)
    {
      target = cgraph_node::get (fndecl);
      if (target)
	{
	  target = target->function_symbol ();
	  i = cgraph_node::local_info (target->decl);
	  cum->call_abi = ix86_function_abi (target->decl);
	}
      else
	cum->call_abi = ix86_function_abi (fndecl);
    }
  else
    cum->call_abi = ix86_function_type_abi (fntype);

  cum->caller = caller;

  /* Set up the number of registers to use for passing arguments.  */
  cum->nregs = ix86_regparm;
  if (TARGET_64BIT)
    {
      cum->nregs = (cum->call_abi == SYSV_ABI
                   ? X86_64_REGPARM_MAX
                   : X86_64_MS_REGPARM_MAX);
    }
  if (TARGET_SSE)
    {
      cum->sse_nregs = SSE_REGPARM_MAX;
      if (TARGET_64BIT)
        {
          cum->sse_nregs = (cum->call_abi == SYSV_ABI
                           ? X86_64_SSE_REGPARM_MAX
                           : X86_64_MS_SSE_REGPARM_MAX);
        }
    }
  if (TARGET_MMX)
    cum->mmx_nregs = MMX_REGPARM_MAX;
  cum->warn_avx512f = true;
  cum->warn_avx = true;
  cum->warn_sse = true;
  cum->warn_mmx = true;

  /* Because type might mismatch in between caller and callee, we need to
     use actual type of function for local calls.
     FIXME: cgraph_analyze can be told to actually record if function uses
     va_start so for local functions maybe_vaarg can be made aggressive
     helping K&R code.
     FIXME: once typesytem is fixed, we won't need this code anymore.  */
  if (i && i->local && i->can_change_signature)
    fntype = TREE_TYPE (target->decl);
  cum->stdarg = stdarg_p (fntype);
  cum->maybe_vaarg = (fntype
		      ? (!prototype_p (fntype) || stdarg_p (fntype))
		      : !libname);

  cum->bnd_regno = FIRST_BND_REG;
  cum->bnds_in_bt = 0;
  cum->force_bnd_pass = 0;
  cum->decl = fndecl;

  if (!TARGET_64BIT)
    {
      /* If there are variable arguments, then we won't pass anything
         in registers in 32-bit mode. */
      if (stdarg_p (fntype))
	{
	  cum->nregs = 0;
	  /* Since in 32-bit, variable arguments are always passed on
	     stack, there is scratch register available for indirect
	     sibcall.  */
	  cfun->machine->arg_reg_available = true;
	  cum->sse_nregs = 0;
	  cum->mmx_nregs = 0;
	  cum->warn_avx512f = false;
	  cum->warn_avx = false;
	  cum->warn_sse = false;
	  cum->warn_mmx = false;
	  return;
	}

      /* Use ecx and edx registers if function has fastcall attribute,
	 else look for regparm information.  */
      if (fntype)
	{
	  unsigned int ccvt = ix86_get_callcvt (fntype);
	  if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
	    {
	      cum->nregs = 1;
	      cum->fastcall = 1; /* Same first register as in fastcall.  */
	    }
	  else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
	    {
	      cum->nregs = 2;
	      cum->fastcall = 1;
	    }
	  else
	    cum->nregs = ix86_function_regparm (fntype, fndecl);
	}

      /* Set up the number of SSE registers used for passing SFmode
	 and DFmode arguments.  Warn for mismatching ABI.  */
      cum->float_in_sse = ix86_function_sseregparm (fntype, fndecl, true);
    }

  cfun->machine->arg_reg_available = (cum->nregs > 0);
}

/* Return the "natural" mode for TYPE.  In most cases, this is just TYPE_MODE.
   But in the case of vector types, it is some vector mode.

   When we have only some of our vector isa extensions enabled, then there
   are some modes for which vector_mode_supported_p is false.  For these
   modes, the generic vector support in gcc will choose some non-vector mode
   in order to implement the type.  By computing the natural mode, we'll
   select the proper ABI location for the operand and not depend on whatever
   the middle-end decides to do with these vector types.

   The midde-end can't deal with the vector types > 16 bytes.  In this
   case, we return the original mode and warn ABI change if CUM isn't
   NULL. 

   If INT_RETURN is true, warn ABI change if the vector mode isn't
   available for function return value.  */

static machine_mode
type_natural_mode (const_tree type, const CUMULATIVE_ARGS *cum,
		   bool in_return)
{
  machine_mode mode = TYPE_MODE (type);

  if (TREE_CODE (type) == VECTOR_TYPE && !VECTOR_MODE_P (mode))
    {
      HOST_WIDE_INT size = int_size_in_bytes (type);
      if ((size == 8 || size == 16 || size == 32 || size == 64)
	  /* ??? Generic code allows us to create width 1 vectors.  Ignore.  */
	  && TYPE_VECTOR_SUBPARTS (type) > 1)
	{
	  machine_mode innermode = TYPE_MODE (TREE_TYPE (type));

	  /* There are no XFmode vector modes.  */
	  if (innermode == XFmode)
	    return mode;

	  if (TREE_CODE (TREE_TYPE (type)) == REAL_TYPE)
	    mode = MIN_MODE_VECTOR_FLOAT;
	  else
	    mode = MIN_MODE_VECTOR_INT;

	  /* Get the mode which has this inner mode and number of units.  */
	  for (; mode != VOIDmode; mode = GET_MODE_WIDER_MODE (mode))
	    if (GET_MODE_NUNITS (mode) == TYPE_VECTOR_SUBPARTS (type)
		&& GET_MODE_INNER (mode) == innermode)
	      {
		if (size == 64 && !TARGET_AVX512F && !TARGET_IAMCU)
		  {
		    static bool warnedavx512f;
		    static bool warnedavx512f_ret;

		    if (cum && cum->warn_avx512f && !warnedavx512f)
		      {
			if (warning (OPT_Wpsabi, "AVX512F vector argument "
				     "without AVX512F enabled changes the ABI"))
			  warnedavx512f = true;
		      }
		    else if (in_return && !warnedavx512f_ret)
		      {
			if (warning (OPT_Wpsabi, "AVX512F vector return "
				     "without AVX512F enabled changes the ABI"))
			  warnedavx512f_ret = true;
		      }

		    return TYPE_MODE (type);
		  }
		else if (size == 32 && !TARGET_AVX && !TARGET_IAMCU)
		  {
		    static bool warnedavx;
		    static bool warnedavx_ret;

		    if (cum && cum->warn_avx && !warnedavx)
		      {
			if (warning (OPT_Wpsabi, "AVX vector argument "
				     "without AVX enabled changes the ABI"))
			  warnedavx = true;
		      }
		    else if (in_return && !warnedavx_ret)
		      {
			if (warning (OPT_Wpsabi, "AVX vector return "
				     "without AVX enabled changes the ABI"))
			  warnedavx_ret = true;
		      }

		    return TYPE_MODE (type);
		  }
		else if (((size == 8 && TARGET_64BIT) || size == 16)
			 && !TARGET_SSE
			 && !TARGET_IAMCU)
		  {
		    static bool warnedsse;
		    static bool warnedsse_ret;

		    if (cum && cum->warn_sse && !warnedsse)
		      {
			if (warning (OPT_Wpsabi, "SSE vector argument "
				     "without SSE enabled changes the ABI"))
			  warnedsse = true;
		      }
		    else if (!TARGET_64BIT && in_return && !warnedsse_ret)
		      {
			if (warning (OPT_Wpsabi, "SSE vector return "
				     "without SSE enabled changes the ABI"))
			  warnedsse_ret = true;
		      }
		  }
		else if ((size == 8 && !TARGET_64BIT)
			 && (!cfun
			     || cfun->machine->func_type == TYPE_NORMAL)
			 && !TARGET_MMX
			 && !TARGET_IAMCU)
		  {
		    static bool warnedmmx;
		    static bool warnedmmx_ret;

		    if (cum && cum->warn_mmx && !warnedmmx)
		      {
			if (warning (OPT_Wpsabi, "MMX vector argument "
				     "without MMX enabled changes the ABI"))
			  warnedmmx = true;
		      }
		    else if (in_return && !warnedmmx_ret)
		      {
			if (warning (OPT_Wpsabi, "MMX vector return "
				     "without MMX enabled changes the ABI"))
			  warnedmmx_ret = true;
		      }
		  }
		return mode;
	      }

	  gcc_unreachable ();
	}
    }

  return mode;
}

/* We want to pass a value in REGNO whose "natural" mode is MODE.  However,
   this may not agree with the mode that the type system has chosen for the
   register, which is ORIG_MODE.  If ORIG_MODE is not BLKmode, then we can
   go ahead and use it.  Otherwise we have to build a PARALLEL instead.  */

static rtx
gen_reg_or_parallel (machine_mode mode, machine_mode orig_mode,
		     unsigned int regno)
{
  rtx tmp;

  if (orig_mode != BLKmode)
    tmp = gen_rtx_REG (orig_mode, regno);
  else
    {
      tmp = gen_rtx_REG (mode, regno);
      tmp = gen_rtx_EXPR_LIST (VOIDmode, tmp, const0_rtx);
      tmp = gen_rtx_PARALLEL (orig_mode, gen_rtvec (1, tmp));
    }

  return tmp;
}

/* x86-64 register passing implementation.  See x86-64 ABI for details.  Goal
   of this code is to classify each 8bytes of incoming argument by the register
   class and assign registers accordingly.  */

/* Return the union class of CLASS1 and CLASS2.
   See the x86-64 PS ABI for details.  */

static enum x86_64_reg_class
merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
{
  /* Rule #1: If both classes are equal, this is the resulting class.  */
  if (class1 == class2)
    return class1;

  /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
     the other class.  */
  if (class1 == X86_64_NO_CLASS)
    return class2;
  if (class2 == X86_64_NO_CLASS)
    return class1;

  /* Rule #3: If one of the classes is MEMORY, the result is MEMORY.  */
  if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
    return X86_64_MEMORY_CLASS;

  /* Rule #4: If one of the classes is INTEGER, the result is INTEGER.  */
  if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
      || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
    return X86_64_INTEGERSI_CLASS;
  if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
      || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
    return X86_64_INTEGER_CLASS;

  /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
     MEMORY is used.  */
  if (class1 == X86_64_X87_CLASS
      || class1 == X86_64_X87UP_CLASS
      || class1 == X86_64_COMPLEX_X87_CLASS
      || class2 == X86_64_X87_CLASS
      || class2 == X86_64_X87UP_CLASS
      || class2 == X86_64_COMPLEX_X87_CLASS)
    return X86_64_MEMORY_CLASS;

  /* Rule #6: Otherwise class SSE is used.  */
  return X86_64_SSE_CLASS;
}

/* Classify the argument of type TYPE and mode MODE.
   CLASSES will be filled by the register class used to pass each word
   of the operand.  The number of words is returned.  In case the parameter
   should be passed in memory, 0 is returned. As a special case for zero
   sized containers, classes[0] will be NO_CLASS and 1 is returned.

   BIT_OFFSET is used internally for handling records and specifies offset
   of the offset in bits modulo 512 to avoid overflow cases.

   See the x86-64 PS ABI for details.
*/

static int
classify_argument (machine_mode mode, const_tree type,
		   enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset)
{
  HOST_WIDE_INT bytes =
    (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
  int words = CEIL (bytes + (bit_offset % 64) / 8, UNITS_PER_WORD);

  /* Variable sized entities are always passed/returned in memory.  */
  if (bytes < 0)
    return 0;

  if (mode != VOIDmode
      && targetm.calls.must_pass_in_stack (mode, type))
    return 0;

  if (type && AGGREGATE_TYPE_P (type))
    {
      int i;
      tree field;
      enum x86_64_reg_class subclasses[MAX_CLASSES];

      /* On x86-64 we pass structures larger than 64 bytes on the stack.  */
      if (bytes > 64)
	return 0;

      for (i = 0; i < words; i++)
	classes[i] = X86_64_NO_CLASS;

      /* Zero sized arrays or structures are NO_CLASS.  We return 0 to
	 signalize memory class, so handle it as special case.  */
      if (!words)
	{
	  classes[0] = X86_64_NO_CLASS;
	  return 1;
	}

      /* Classify each field of record and merge classes.  */
      switch (TREE_CODE (type))
	{
	case RECORD_TYPE:
	  /* And now merge the fields of structure.  */
	  for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
	    {
	      if (TREE_CODE (field) == FIELD_DECL)
		{
		  int num;

		  if (TREE_TYPE (field) == error_mark_node)
		    continue;

		  /* Bitfields are always classified as integer.  Handle them
		     early, since later code would consider them to be
		     misaligned integers.  */
		  if (DECL_BIT_FIELD (field))
		    {
		      for (i = (int_bit_position (field)
				+ (bit_offset % 64)) / 8 / 8;
			   i < ((int_bit_position (field) + (bit_offset % 64))
			        + tree_to_shwi (DECL_SIZE (field))
				+ 63) / 8 / 8; i++)
			classes[i] =
			  merge_classes (X86_64_INTEGER_CLASS,
					 classes[i]);
		    }
		  else
		    {
		      int pos;

		      type = TREE_TYPE (field);

		      /* Flexible array member is ignored.  */
		      if (TYPE_MODE (type) == BLKmode
			  && TREE_CODE (type) == ARRAY_TYPE
			  && TYPE_SIZE (type) == NULL_TREE
			  && TYPE_DOMAIN (type) != NULL_TREE
			  && (TYPE_MAX_VALUE (TYPE_DOMAIN (type))
			      == NULL_TREE))
			{
			  static bool warned;

			  if (!warned && warn_psabi)
			    {
			      warned = true;
			      inform (input_location,
				      "the ABI of passing struct with"
				      " a flexible array member has"
				      " changed in GCC 4.4");
			    }
			  continue;
			}
		      num = classify_argument (TYPE_MODE (type), type,
					       subclasses,
					       (int_bit_position (field)
						+ bit_offset) % 512);
		      if (!num)
			return 0;
		      pos = (int_bit_position (field)
			     + (bit_offset % 64)) / 8 / 8;
		      for (i = 0; i < num && (i + pos) < words; i++)
			classes[i + pos] =
			  merge_classes (subclasses[i], classes[i + pos]);
		    }
		}
	    }
	  break;

	case ARRAY_TYPE:
	  /* Arrays are handled as small records.  */
	  {
	    int num;
	    num = classify_argument (TYPE_MODE (TREE_TYPE (type)),
				     TREE_TYPE (type), subclasses, bit_offset);
	    if (!num)
	      return 0;

	    /* The partial classes are now full classes.  */
	    if (subclasses[0] == X86_64_SSESF_CLASS && bytes != 4)
	      subclasses[0] = X86_64_SSE_CLASS;
	    if (subclasses[0] == X86_64_INTEGERSI_CLASS
		&& !((bit_offset % 64) == 0 && bytes == 4))
	      subclasses[0] = X86_64_INTEGER_CLASS;

	    for (i = 0; i < words; i++)
	      classes[i] = subclasses[i % num];

	    break;
	  }
	case UNION_TYPE:
	case QUAL_UNION_TYPE:
	  /* Unions are similar to RECORD_TYPE but offset is always 0.
	     */
	  for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
	    {
	      if (TREE_CODE (field) == FIELD_DECL)
		{
		  int num;

		  if (TREE_TYPE (field) == error_mark_node)
		    continue;

		  num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
					   TREE_TYPE (field), subclasses,
					   bit_offset);
		  if (!num)
		    return 0;
		  for (i = 0; i < num && i < words; i++)
		    classes[i] = merge_classes (subclasses[i], classes[i]);
		}
	    }
	  break;

	default:
	  gcc_unreachable ();
	}

      if (words > 2)
	{
	  /* When size > 16 bytes, if the first one isn't
	     X86_64_SSE_CLASS or any other ones aren't
	     X86_64_SSEUP_CLASS, everything should be passed in
	     memory.  */
	  if (classes[0] != X86_64_SSE_CLASS)
	      return 0;

	  for (i = 1; i < words; i++)
	    if (classes[i] != X86_64_SSEUP_CLASS)
	      return 0;
	}

      /* Final merger cleanup.  */
      for (i = 0; i < words; i++)
	{
	  /* If one class is MEMORY, everything should be passed in
	     memory.  */
	  if (classes[i] == X86_64_MEMORY_CLASS)
	    return 0;

	  /* The X86_64_SSEUP_CLASS should be always preceded by
	     X86_64_SSE_CLASS or X86_64_SSEUP_CLASS.  */
	  if (classes[i] == X86_64_SSEUP_CLASS
	      && classes[i - 1] != X86_64_SSE_CLASS
	      && classes[i - 1] != X86_64_SSEUP_CLASS)
	    {
	      /* The first one should never be X86_64_SSEUP_CLASS.  */
	      gcc_assert (i != 0);
	      classes[i] = X86_64_SSE_CLASS;
	    }

	  /*  If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
	       everything should be passed in memory.  */
	  if (classes[i] == X86_64_X87UP_CLASS
	      && (classes[i - 1] != X86_64_X87_CLASS))
	    {
	      static bool warned;

	      /* The first one should never be X86_64_X87UP_CLASS.  */
	      gcc_assert (i != 0);
	      if (!warned && warn_psabi)
		{
		  warned = true;
		  inform (input_location,
			  "the ABI of passing union with long double"
			  " has changed in GCC 4.4");
		}
	      return 0;
	    }
	}
      return words;
    }

  /* Compute alignment needed.  We align all types to natural boundaries with
     exception of XFmode that is aligned to 64bits.  */
  if (mode != VOIDmode && mode != BLKmode)
    {
      int mode_alignment = GET_MODE_BITSIZE (mode);

      if (mode == XFmode)
	mode_alignment = 128;
      else if (mode == XCmode)
	mode_alignment = 256;
      if (COMPLEX_MODE_P (mode))
	mode_alignment /= 2;
      /* Misaligned fields are always returned in memory.  */
      if (bit_offset % mode_alignment)
	return 0;
    }

  /* for V1xx modes, just use the base mode */
  if (VECTOR_MODE_P (mode) && mode != V1DImode && mode != V1TImode
      && GET_MODE_UNIT_SIZE (mode) == bytes)
    mode = GET_MODE_INNER (mode);

  /* Classification of atomic types.  */
  switch (mode)
    {
    case SDmode:
    case DDmode:
      classes[0] = X86_64_SSE_CLASS;
      return 1;
    case TDmode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      return 2;
    case DImode:
    case SImode:
    case HImode:
    case QImode:
    case CSImode:
    case CHImode:
    case CQImode:
      {
	int size = bit_offset + (int) GET_MODE_BITSIZE (mode);

	/* Analyze last 128 bits only.  */
	size = (size - 1) & 0x7f;

	if (size < 32)
	  {
	    classes[0] = X86_64_INTEGERSI_CLASS;
	    return 1;
	  }
	else if (size < 64)
	  {
	    classes[0] = X86_64_INTEGER_CLASS;
	    return 1;
	  }
	else if (size < 64+32)
	  {
	    classes[0] = X86_64_INTEGER_CLASS;
	    classes[1] = X86_64_INTEGERSI_CLASS;
	    return 2;
	  }
	else if (size < 64+64)
	  {
	    classes[0] = classes[1] = X86_64_INTEGER_CLASS;
	    return 2;
	  }
	else
	  gcc_unreachable ();
      }
    case CDImode:
    case TImode:
      classes[0] = classes[1] = X86_64_INTEGER_CLASS;
      return 2;
    case COImode:
    case OImode:
      /* OImode shouldn't be used directly.  */
      gcc_unreachable ();
    case CTImode:
      return 0;
    case SFmode:
      if (!(bit_offset % 64))
	classes[0] = X86_64_SSESF_CLASS;
      else
	classes[0] = X86_64_SSE_CLASS;
      return 1;
    case DFmode:
      classes[0] = X86_64_SSEDF_CLASS;
      return 1;
    case XFmode:
      classes[0] = X86_64_X87_CLASS;
      classes[1] = X86_64_X87UP_CLASS;
      return 2;
    case TFmode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      return 2;
    case SCmode:
      classes[0] = X86_64_SSE_CLASS;
      if (!(bit_offset % 64))
	return 1;
      else
	{
	  static bool warned;

	  if (!warned && warn_psabi)
	    {
	      warned = true;
	      inform (input_location,
		      "the ABI of passing structure with complex float"
		      " member has changed in GCC 4.4");
	    }
	  classes[1] = X86_64_SSESF_CLASS;
	  return 2;
	}
    case DCmode:
      classes[0] = X86_64_SSEDF_CLASS;
      classes[1] = X86_64_SSEDF_CLASS;
      return 2;
    case XCmode:
      classes[0] = X86_64_COMPLEX_X87_CLASS;
      return 1;
    case TCmode:
      /* This modes is larger than 16 bytes.  */
      return 0;
    case V8SFmode:
    case V8SImode:
    case V32QImode:
    case V16HImode:
    case V4DFmode:
    case V4DImode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      classes[2] = X86_64_SSEUP_CLASS;
      classes[3] = X86_64_SSEUP_CLASS;
      return 4;
    case V8DFmode:
    case V16SFmode:
    case V8DImode:
    case V16SImode:
    case V32HImode:
    case V64QImode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      classes[2] = X86_64_SSEUP_CLASS;
      classes[3] = X86_64_SSEUP_CLASS;
      classes[4] = X86_64_SSEUP_CLASS;
      classes[5] = X86_64_SSEUP_CLASS;
      classes[6] = X86_64_SSEUP_CLASS;
      classes[7] = X86_64_SSEUP_CLASS;
      return 8;
    case V4SFmode:
    case V4SImode:
    case V16QImode:
    case V8HImode:
    case V2DFmode:
    case V2DImode:
      classes[0] = X86_64_SSE_CLASS;
      classes[1] = X86_64_SSEUP_CLASS;
      return 2;
    case V1TImode:
    case V1DImode:
    case V2SFmode:
    case V2SImode:
    case V4HImode:
    case V8QImode:
      classes[0] = X86_64_SSE_CLASS;
      return 1;
    case BLKmode:
    case VOIDmode:
      return 0;
    default:
      gcc_assert (VECTOR_MODE_P (mode));

      if (bytes > 16)
	return 0;

      gcc_assert (GET_MODE_CLASS (GET_MODE_INNER (mode)) == MODE_INT);

      if (bit_offset + GET_MODE_BITSIZE (mode) <= 32)
	classes[0] = X86_64_INTEGERSI_CLASS;
      else
	classes[0] = X86_64_INTEGER_CLASS;
      classes[1] = X86_64_INTEGER_CLASS;
      return 1 + (bytes > 8);
    }
}

/* Examine the argument and return set number of register required in each
   class.  Return true iff parameter should be passed in memory.  */

static bool
examine_argument (machine_mode mode, const_tree type, int in_return,
		  int *int_nregs, int *sse_nregs)
{
  enum x86_64_reg_class regclass[MAX_CLASSES];
  int n = classify_argument (mode, type, regclass, 0);

  *int_nregs = 0;
  *sse_nregs = 0;

  if (!n)
    return true;
  for (n--; n >= 0; n--)
    switch (regclass[n])
      {
      case X86_64_INTEGER_CLASS:
      case X86_64_INTEGERSI_CLASS:
	(*int_nregs)++;
	break;
      case X86_64_SSE_CLASS:
      case X86_64_SSESF_CLASS:
      case X86_64_SSEDF_CLASS:
	(*sse_nregs)++;
	break;
      case X86_64_NO_CLASS:
      case X86_64_SSEUP_CLASS:
	break;
      case X86_64_X87_CLASS:
      case X86_64_X87UP_CLASS:
      case X86_64_COMPLEX_X87_CLASS:
	if (!in_return)
	  return true;
	break;
      case X86_64_MEMORY_CLASS:
	gcc_unreachable ();
      }

  return false;
}

/* Construct container for the argument used by GCC interface.  See
   FUNCTION_ARG for the detailed description.  */

static rtx
construct_container (machine_mode mode, machine_mode orig_mode,
		     const_tree type, int in_return, int nintregs, int nsseregs,
		     const int *intreg, int sse_regno)
{
  /* The following variables hold the static issued_error state.  */
  static bool issued_sse_arg_error;
  static bool issued_sse_ret_error;
  static bool issued_x87_ret_error;

  machine_mode tmpmode;
  int bytes =
    (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
  enum x86_64_reg_class regclass[MAX_CLASSES];
  int n;
  int i;
  int nexps = 0;
  int needed_sseregs, needed_intregs;
  rtx exp[MAX_CLASSES];
  rtx ret;

  n = classify_argument (mode, type, regclass, 0);
  if (!n)
    return NULL;
  if (examine_argument (mode, type, in_return, &needed_intregs,
			&needed_sseregs))
    return NULL;
  if (needed_intregs > nintregs || needed_sseregs > nsseregs)
    return NULL;

  /* We allowed the user to turn off SSE for kernel mode.  Don't crash if
     some less clueful developer tries to use floating-point anyway.  */
  if (needed_sseregs && !TARGET_SSE)
    {
      if (in_return)
	{
	  if (!issued_sse_ret_error)
	    {
	      error ("SSE register return with SSE disabled");
	      issued_sse_ret_error = true;
	    }
	}
      else if (!issued_sse_arg_error)
	{
	  error ("SSE register argument with SSE disabled");
	  issued_sse_arg_error = true;
	}
      return NULL;
    }

  /* Likewise, error if the ABI requires us to return values in the
     x87 registers and the user specified -mno-80387.  */
  if (!TARGET_FLOAT_RETURNS_IN_80387 && in_return)
    for (i = 0; i < n; i++)
      if (regclass[i] == X86_64_X87_CLASS
	  || regclass[i] == X86_64_X87UP_CLASS
	  || regclass[i] == X86_64_COMPLEX_X87_CLASS)
	{
	  if (!issued_x87_ret_error)
	    {
	      error ("x87 register return with x87 disabled");
	      issued_x87_ret_error = true;
	    }
	  return NULL;
	}

  /* First construct simple cases.  Avoid SCmode, since we want to use
     single register to pass this type.  */
  if (n == 1 && mode != SCmode)
    switch (regclass[0])
      {
      case X86_64_INTEGER_CLASS:
      case X86_64_INTEGERSI_CLASS:
	return gen_rtx_REG (mode, intreg[0]);
      case X86_64_SSE_CLASS:
      case X86_64_SSESF_CLASS:
      case X86_64_SSEDF_CLASS:
	if (mode != BLKmode)
	  return gen_reg_or_parallel (mode, orig_mode,
				      SSE_REGNO (sse_regno));
	break;
      case X86_64_X87_CLASS:
      case X86_64_COMPLEX_X87_CLASS:
	return gen_rtx_REG (mode, FIRST_STACK_REG);
      case X86_64_NO_CLASS:
	/* Zero sized array, struct or class.  */
	return NULL;
      default:
	gcc_unreachable ();
      }
  if (n == 2
      && regclass[0] == X86_64_SSE_CLASS
      && regclass[1] == X86_64_SSEUP_CLASS
      && mode != BLKmode)
    return gen_reg_or_parallel (mode, orig_mode,
				SSE_REGNO (sse_regno));
  if (n == 4
      && regclass[0] == X86_64_SSE_CLASS
      && regclass[1] == X86_64_SSEUP_CLASS
      && regclass[2] == X86_64_SSEUP_CLASS
      && regclass[3] == X86_64_SSEUP_CLASS
      && mode != BLKmode)
    return gen_reg_or_parallel (mode, orig_mode,
				SSE_REGNO (sse_regno));
  if (n == 8
      && regclass[0] == X86_64_SSE_CLASS
      && regclass[1] == X86_64_SSEUP_CLASS
      && regclass[2] == X86_64_SSEUP_CLASS
      && regclass[3] == X86_64_SSEUP_CLASS
      && regclass[4] == X86_64_SSEUP_CLASS
      && regclass[5] == X86_64_SSEUP_CLASS
      && regclass[6] == X86_64_SSEUP_CLASS
      && regclass[7] == X86_64_SSEUP_CLASS
      && mode != BLKmode)
    return gen_reg_or_parallel (mode, orig_mode,
				SSE_REGNO (sse_regno));
  if (n == 2
      && regclass[0] == X86_64_X87_CLASS
      && regclass[1] == X86_64_X87UP_CLASS)
    return gen_rtx_REG (XFmode, FIRST_STACK_REG);

  if (n == 2
      && regclass[0] == X86_64_INTEGER_CLASS
      && regclass[1] == X86_64_INTEGER_CLASS
      && (mode == CDImode || mode == TImode)
      && intreg[0] + 1 == intreg[1])
    return gen_rtx_REG (mode, intreg[0]);

  /* Otherwise figure out the entries of the PARALLEL.  */
  for (i = 0; i < n; i++)
    {
      int pos;

      switch (regclass[i])
        {
	  case X86_64_NO_CLASS:
	    break;
	  case X86_64_INTEGER_CLASS:
	  case X86_64_INTEGERSI_CLASS:
	    /* Merge TImodes on aligned occasions here too.  */
	    if (i * 8 + 8 > bytes)
	      tmpmode
		= mode_for_size ((bytes - i * 8) * BITS_PER_UNIT, MODE_INT, 0);
	    else if (regclass[i] == X86_64_INTEGERSI_CLASS)
	      tmpmode = SImode;
	    else
	      tmpmode = DImode;
	    /* We've requested 24 bytes we
	       don't have mode for.  Use DImode.  */
	    if (tmpmode == BLKmode)
	      tmpmode = DImode;
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (tmpmode, *intreg),
				   GEN_INT (i*8));
	    intreg++;
	    break;
	  case X86_64_SSESF_CLASS:
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (SFmode,
						SSE_REGNO (sse_regno)),
				   GEN_INT (i*8));
	    sse_regno++;
	    break;
	  case X86_64_SSEDF_CLASS:
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (DFmode,
						SSE_REGNO (sse_regno)),
				   GEN_INT (i*8));
	    sse_regno++;
	    break;
	  case X86_64_SSE_CLASS:
	    pos = i;
	    switch (n)
	      {
	      case 1:
		tmpmode = DImode;
		break;
	      case 2:
		if (i == 0 && regclass[1] == X86_64_SSEUP_CLASS)
		  {
		    tmpmode = TImode;
		    i++;
		  }
		else
		  tmpmode = DImode;
		break;
	      case 4:
		gcc_assert (i == 0
			    && regclass[1] == X86_64_SSEUP_CLASS
			    && regclass[2] == X86_64_SSEUP_CLASS
			    && regclass[3] == X86_64_SSEUP_CLASS);
		tmpmode = OImode;
		i += 3;
		break;
	      case 8:
		gcc_assert (i == 0
			    && regclass[1] == X86_64_SSEUP_CLASS
			    && regclass[2] == X86_64_SSEUP_CLASS
			    && regclass[3] == X86_64_SSEUP_CLASS
			    && regclass[4] == X86_64_SSEUP_CLASS
			    && regclass[5] == X86_64_SSEUP_CLASS
			    && regclass[6] == X86_64_SSEUP_CLASS
			    && regclass[7] == X86_64_SSEUP_CLASS);
		tmpmode = XImode;
		i += 7;
		break;
	      default:
		gcc_unreachable ();
	      }
	    exp [nexps++]
	      = gen_rtx_EXPR_LIST (VOIDmode,
				   gen_rtx_REG (tmpmode,
						SSE_REGNO (sse_regno)),
				   GEN_INT (pos*8));
	    sse_regno++;
	    break;
	  default:
	    gcc_unreachable ();
	}
    }

  /* Empty aligned struct, union or class.  */
  if (nexps == 0)
    return NULL;

  ret =  gen_rtx_PARALLEL (mode, rtvec_alloc (nexps));
  for (i = 0; i < nexps; i++)
    XVECEXP (ret, 0, i) = exp [i];
  return ret;
}

/* Update the data in CUM to advance over an argument of mode MODE
   and data type TYPE.  (TYPE is null for libcalls where that information
   may not be available.)

   Return a number of integer regsiters advanced over.  */

static int
function_arg_advance_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
			 const_tree type, HOST_WIDE_INT bytes,
			 HOST_WIDE_INT words)
{
  int res = 0;
  bool error_p = NULL;

  if (TARGET_IAMCU)
    {
      /* Intel MCU psABI passes scalars and aggregates no larger than 8
	 bytes in registers.  */
      if (!VECTOR_MODE_P (mode) && bytes <= 8)
	goto pass_in_reg;
      return res;
    }

  switch (mode)
    {
    default:
      break;

    case BLKmode:
      if (bytes < 0)
	break;
      /* FALLTHRU */

    case DImode:
    case SImode:
    case HImode:
    case QImode:
pass_in_reg:
      cum->words += words;
      cum->nregs -= words;
      cum->regno += words;
      if (cum->nregs >= 0)
	res = words;
      if (cum->nregs <= 0)
	{
	  cum->nregs = 0;
	  cfun->machine->arg_reg_available = false;
	  cum->regno = 0;
	}
      break;

    case OImode:
      /* OImode shouldn't be used directly.  */
      gcc_unreachable ();

    case DFmode:
      if (cum->float_in_sse == -1)
	error_p = 1;
      if (cum->float_in_sse < 2)
	break;
      /* FALLTHRU */
    case SFmode:
      if (cum->float_in_sse == -1)
	error_p = 1;
      if (cum->float_in_sse < 1)
	break;
      /* FALLTHRU */

    case V8SFmode:
    case V8SImode:
    case V64QImode:
    case V32HImode:
    case V16SImode:
    case V8DImode:
    case V16SFmode:
    case V8DFmode:
    case V32QImode:
    case V16HImode:
    case V4DFmode:
    case V4DImode:
    case TImode:
    case V16QImode:
    case V8HImode:
    case V4SImode:
    case V2DImode:
    case V4SFmode:
    case V2DFmode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  cum->sse_words += words;
	  cum->sse_nregs -= 1;
	  cum->sse_regno += 1;
	  if (cum->sse_nregs <= 0)
	    {
	      cum->sse_nregs = 0;
	      cum->sse_regno = 0;
	    }
	}
      break;

    case V8QImode:
    case V4HImode:
    case V2SImode:
    case V2SFmode:
    case V1TImode:
    case V1DImode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  cum->mmx_words += words;
	  cum->mmx_nregs -= 1;
	  cum->mmx_regno += 1;
	  if (cum->mmx_nregs <= 0)
	    {
	      cum->mmx_nregs = 0;
	      cum->mmx_regno = 0;
	    }
	}
      break;
    }
  if (error_p)
    {
      cum->float_in_sse = 0;
      error ("calling %qD with SSE calling convention without "
	     "SSE/SSE2 enabled", cum->decl);
      sorry ("this is a GCC bug that can be worked around by adding "
	     "attribute used to function called");
    }

  return res;
}

static int
function_arg_advance_64 (CUMULATIVE_ARGS *cum, machine_mode mode,
			 const_tree type, HOST_WIDE_INT words, bool named)
{
  int int_nregs, sse_nregs;

  /* Unnamed 512 and 256bit vector mode parameters are passed on stack.  */
  if (!named && (VALID_AVX512F_REG_MODE (mode)
		 || VALID_AVX256_REG_MODE (mode)))
    return 0;

  if (!examine_argument (mode, type, 0, &int_nregs, &sse_nregs)
      && sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs)
    {
      cum->nregs -= int_nregs;
      cum->sse_nregs -= sse_nregs;
      cum->regno += int_nregs;
      cum->sse_regno += sse_nregs;
      return int_nregs;
    }
  else
    {
      int align = ix86_function_arg_boundary (mode, type) / BITS_PER_WORD;
      cum->words = ROUND_UP (cum->words, align);
      cum->words += words;
      return 0;
    }
}

static int
function_arg_advance_ms_64 (CUMULATIVE_ARGS *cum, HOST_WIDE_INT bytes,
			    HOST_WIDE_INT words)
{
  /* Otherwise, this should be passed indirect.  */
  gcc_assert (bytes == 1 || bytes == 2 || bytes == 4 || bytes == 8);

  cum->words += words;
  if (cum->nregs > 0)
    {
      cum->nregs -= 1;
      cum->regno += 1;
      return 1;
    }
  return 0;
}

/* Update the data in CUM to advance over an argument of mode MODE and
   data type TYPE.  (TYPE is null for libcalls where that information
   may not be available.)  */

static void
ix86_function_arg_advance (cumulative_args_t cum_v, machine_mode mode,
			   const_tree type, bool named)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
  HOST_WIDE_INT bytes, words;
  int nregs;

  /* The argument of interrupt handler is a special case and is
     handled in ix86_function_arg.  */
  if (!cum->caller && cfun->machine->func_type != TYPE_NORMAL)
    return;

  if (mode == BLKmode)
    bytes = int_size_in_bytes (type);
  else
    bytes = GET_MODE_SIZE (mode);
  words = CEIL (bytes, UNITS_PER_WORD);

  if (type)
    mode = type_natural_mode (type, NULL, false);

  if ((type && POINTER_BOUNDS_TYPE_P (type))
      || POINTER_BOUNDS_MODE_P (mode))
    {
      /* If we pass bounds in BT then just update remained bounds count.  */
      if (cum->bnds_in_bt)
	{
	  cum->bnds_in_bt--;
	  return;
	}

      /* Update remained number of bounds to force.  */
      if (cum->force_bnd_pass)
	cum->force_bnd_pass--;

      cum->bnd_regno++;

      return;
    }

  /* The first arg not going to Bounds Tables resets this counter.  */
  cum->bnds_in_bt = 0;
  /* For unnamed args we always pass bounds to avoid bounds mess when
     passed and received types do not match.  If bounds do not follow
     unnamed arg, still pretend required number of bounds were passed.  */
  if (cum->force_bnd_pass)
    {
      cum->bnd_regno += cum->force_bnd_pass;
      cum->force_bnd_pass = 0;
    }

  if (TARGET_64BIT)
    {
      enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;

      if (call_abi == MS_ABI)
	nregs = function_arg_advance_ms_64 (cum, bytes, words);
      else
	nregs = function_arg_advance_64 (cum, mode, type, words, named);
    }
  else
    nregs = function_arg_advance_32 (cum, mode, type, bytes, words);

  /* For stdarg we expect bounds to be passed for each value passed
     in register.  */
  if (cum->stdarg)
    cum->force_bnd_pass = nregs;
  /* For pointers passed in memory we expect bounds passed in Bounds
     Table.  */
  if (!nregs)
    cum->bnds_in_bt = chkp_type_bounds_count (type);
}

/* Define where to put the arguments to a function.
   Value is zero to push the argument on the stack,
   or a hard register in which to store the argument.

   MODE is the argument's machine mode.
   TYPE is the data type of the argument (as a tree).
    This is null for libcalls where that information may
    not be available.
   CUM is a variable of type CUMULATIVE_ARGS which gives info about
    the preceding args and about the function being called.
   NAMED is nonzero if this argument is a named parameter
    (otherwise it is an extra parameter matching an ellipsis).  */

static rtx
function_arg_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
		 machine_mode orig_mode, const_tree type,
		 HOST_WIDE_INT bytes, HOST_WIDE_INT words)
{
  bool error_p = false;
  /* Avoid the AL settings for the Unix64 ABI.  */
  if (mode == VOIDmode)
    return constm1_rtx;

  if (TARGET_IAMCU)
    {
      /* Intel MCU psABI passes scalars and aggregates no larger than 8
	 bytes in registers.  */
      if (!VECTOR_MODE_P (mode) && bytes <= 8)
	goto pass_in_reg;
      return NULL_RTX;
    }

  switch (mode)
    {
    default:
      break;

    case BLKmode:
      if (bytes < 0)
	break;
      /* FALLTHRU */
    case DImode:
    case SImode:
    case HImode:
    case QImode:
pass_in_reg:
      if (words <= cum->nregs)
	{
	  int regno = cum->regno;

	  /* Fastcall allocates the first two DWORD (SImode) or
            smaller arguments to ECX and EDX if it isn't an
            aggregate type .  */
	  if (cum->fastcall)
	    {
	      if (mode == BLKmode
		  || mode == DImode
		  || (type && AGGREGATE_TYPE_P (type)))
	        break;

	      /* ECX not EAX is the first allocated register.  */
	      if (regno == AX_REG)
		regno = CX_REG;
	    }
	  return gen_rtx_REG (mode, regno);
	}
      break;

    case DFmode:
      if (cum->float_in_sse == -1)
	error_p = 1;
      if (cum->float_in_sse < 2)
	break;
      /* FALLTHRU */
    case SFmode:
      if (cum->float_in_sse == -1)
	error_p = 1;
      if (cum->float_in_sse < 1)
	break;
      /* FALLTHRU */
    case TImode:
      /* In 32bit, we pass TImode in xmm registers.  */
    case V16QImode:
    case V8HImode:
    case V4SImode:
    case V2DImode:
    case V4SFmode:
    case V2DFmode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  if (cum->sse_nregs)
	    return gen_reg_or_parallel (mode, orig_mode,
				        cum->sse_regno + FIRST_SSE_REG);
	}
      break;

    case OImode:
    case XImode:
      /* OImode and XImode shouldn't be used directly.  */
      gcc_unreachable ();

    case V64QImode:
    case V32HImode:
    case V16SImode:
    case V8DImode:
    case V16SFmode:
    case V8DFmode:
    case V8SFmode:
    case V8SImode:
    case V32QImode:
    case V16HImode:
    case V4DFmode:
    case V4DImode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  if (cum->sse_nregs)
	    return gen_reg_or_parallel (mode, orig_mode,
				        cum->sse_regno + FIRST_SSE_REG);
	}
      break;

    case V8QImode:
    case V4HImode:
    case V2SImode:
    case V2SFmode:
    case V1TImode:
    case V1DImode:
      if (!type || !AGGREGATE_TYPE_P (type))
	{
	  if (cum->mmx_nregs)
	    return gen_reg_or_parallel (mode, orig_mode,
				        cum->mmx_regno + FIRST_MMX_REG);
	}
      break;
    }
  if (error_p)
    {
      cum->float_in_sse = 0;
      error ("calling %qD with SSE calling convention without "
	     "SSE/SSE2 enabled", cum->decl);
      sorry ("this is a GCC bug that can be worked around by adding "
	     "attribute used to function called");
    }

  return NULL_RTX;
}

static rtx
function_arg_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
		 machine_mode orig_mode, const_tree type, bool named)
{
  /* Handle a hidden AL argument containing number of registers
     for varargs x86-64 functions.  */
  if (mode == VOIDmode)
    return GEN_INT (cum->maybe_vaarg
		    ? (cum->sse_nregs < 0
		       ? X86_64_SSE_REGPARM_MAX
		       : cum->sse_regno)
		    : -1);

  switch (mode)
    {
    default:
      break;

    case V8SFmode:
    case V8SImode:
    case V32QImode:
    case V16HImode:
    case V4DFmode:
    case V4DImode:
    case V16SFmode:
    case V16SImode:
    case V64QImode:
    case V32HImode:
    case V8DFmode:
    case V8DImode:
      /* Unnamed 256 and 512bit vector mode parameters are passed on stack.  */
      if (!named)
	return NULL;
      break;
    }

  return construct_container (mode, orig_mode, type, 0, cum->nregs,
			      cum->sse_nregs,
			      &x86_64_int_parameter_registers [cum->regno],
			      cum->sse_regno);
}

static rtx
function_arg_ms_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
		    machine_mode orig_mode, bool named,
		    HOST_WIDE_INT bytes)
{
  unsigned int regno;

  /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
     We use value of -2 to specify that current function call is MSABI.  */
  if (mode == VOIDmode)
    return GEN_INT (-2);

  /* If we've run out of registers, it goes on the stack.  */
  if (cum->nregs == 0)
    return NULL_RTX;

  regno = x86_64_ms_abi_int_parameter_registers[cum->regno];

  /* Only floating point modes are passed in anything but integer regs.  */
  if (TARGET_SSE && (mode == SFmode || mode == DFmode))
    {
      if (named)
	regno = cum->regno + FIRST_SSE_REG;
      else
	{
	  rtx t1, t2;

	  /* Unnamed floating parameters are passed in both the
	     SSE and integer registers.  */
	  t1 = gen_rtx_REG (mode, cum->regno + FIRST_SSE_REG);
	  t2 = gen_rtx_REG (mode, regno);
	  t1 = gen_rtx_EXPR_LIST (VOIDmode, t1, const0_rtx);
	  t2 = gen_rtx_EXPR_LIST (VOIDmode, t2, const0_rtx);
	  return gen_rtx_PARALLEL (mode, gen_rtvec (2, t1, t2));
	}
    }
  /* Handle aggregated types passed in register.  */
  if (orig_mode == BLKmode)
    {
      if (bytes > 0 && bytes <= 8)
        mode = (bytes > 4 ? DImode : SImode);
      if (mode == BLKmode)
        mode = DImode;
    }

  return gen_reg_or_parallel (mode, orig_mode, regno);
}

/* Return where to put the arguments to a function.
   Return zero to push the argument on the stack, or a hard register in which to store the argument.

   MODE is the argument's machine mode.  TYPE is the data type of the
   argument.  It is null for libcalls where that information may not be
   available.  CUM gives information about the preceding args and about
   the function being called.  NAMED is nonzero if this argument is a
   named parameter (otherwise it is an extra parameter matching an
   ellipsis).  */

static rtx
ix86_function_arg (cumulative_args_t cum_v, machine_mode omode,
		   const_tree type, bool named)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
  machine_mode mode = omode;
  HOST_WIDE_INT bytes, words;
  rtx arg;

  if (!cum->caller && cfun->machine->func_type != TYPE_NORMAL)
    {
      gcc_assert (type != NULL_TREE);
      if (POINTER_TYPE_P (type))
	{
	  /* This is the pointer argument.  */
	  gcc_assert (TYPE_MODE (type) == Pmode);
	  if (cfun->machine->func_type == TYPE_INTERRUPT)
	    /* -WORD(AP) in the current frame in interrupt handler.  */
	    arg = plus_constant (Pmode, arg_pointer_rtx,
				 -UNITS_PER_WORD);
	  else
	    /* (AP) in the current frame in exception handler.  */
	    arg = arg_pointer_rtx;
	}
      else
	{
	  gcc_assert (cfun->machine->func_type == TYPE_EXCEPTION
		      && TREE_CODE (type) == INTEGER_TYPE
		      && TYPE_MODE (type) == word_mode);
	  /* The integer argument is the error code at -WORD(AP) in
	     the current frame in exception handler.  */
	  arg = gen_rtx_MEM (word_mode,
			     plus_constant (Pmode,
					    arg_pointer_rtx,
					    -UNITS_PER_WORD));
	}
      return arg;
    }

  /* All pointer bounds arguments are handled separately here.  */
  if ((type && POINTER_BOUNDS_TYPE_P (type))
      || POINTER_BOUNDS_MODE_P (mode))
    {
      /* Return NULL if bounds are forced to go in Bounds Table.  */
      if (cum->bnds_in_bt)
	arg = NULL;
      /* Return the next available bound reg if any.  */
      else if (cum->bnd_regno <= LAST_BND_REG)
	arg = gen_rtx_REG (BNDmode, cum->bnd_regno);
      /* Return the next special slot number otherwise.  */
      else
	arg = GEN_INT (cum->bnd_regno - LAST_BND_REG - 1);

      return arg;
    }

  if (mode == BLKmode)
    bytes = int_size_in_bytes (type);
  else
    bytes = GET_MODE_SIZE (mode);
  words = CEIL (bytes, UNITS_PER_WORD);

  /* To simplify the code below, represent vector types with a vector mode
     even if MMX/SSE are not active.  */
  if (type && TREE_CODE (type) == VECTOR_TYPE)
    mode = type_natural_mode (type, cum, false);

  if (TARGET_64BIT)
    {
      enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;

      if (call_abi == MS_ABI)
	arg = function_arg_ms_64 (cum, mode, omode, named, bytes);
      else
	arg = function_arg_64 (cum, mode, omode, type, named);
    }
  else
    arg = function_arg_32 (cum, mode, omode, type, bytes, words);

  return arg;
}

/* A C expression that indicates when an argument must be passed by
   reference.  If nonzero for an argument, a copy of that argument is
   made in memory and a pointer to the argument is passed instead of
   the argument itself.  The pointer is passed in whatever way is
   appropriate for passing a pointer to that type.  */

static bool
ix86_pass_by_reference (cumulative_args_t cum_v, machine_mode mode,
			const_tree type, bool)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);

  /* Bounds are never passed by reference.  */
  if ((type && POINTER_BOUNDS_TYPE_P (type))
      || POINTER_BOUNDS_MODE_P (mode))
    return false;

  if (TARGET_64BIT)
    {
      enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;

      /* See Windows x64 Software Convention.  */
      if (call_abi == MS_ABI)
	{
	  HOST_WIDE_INT msize = GET_MODE_SIZE (mode);

	  if (type)
	    {
	      /* Arrays are passed by reference.  */
	      if (TREE_CODE (type) == ARRAY_TYPE)
		return true;

	      if (RECORD_OR_UNION_TYPE_P (type))
		{
		  /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
		     are passed by reference.  */
		  msize = int_size_in_bytes (type);
		}
	    }

	  /* __m128 is passed by reference.  */
	  return msize != 1 && msize != 2 && msize != 4 && msize != 8;
	}
      else if (type && int_size_in_bytes (type) == -1)
	return true;
    }

  return false;
}

/* Return true when TYPE should be 128bit aligned for 32bit argument
   passing ABI.  XXX: This function is obsolete and is only used for
   checking psABI compatibility with previous versions of GCC.  */

static bool
ix86_compat_aligned_value_p (const_tree type)
{
  machine_mode mode = TYPE_MODE (type);
  if (((TARGET_SSE && SSE_REG_MODE_P (mode))
       || mode == TDmode
       || mode == TFmode
       || mode == TCmode)
      && (!TYPE_USER_ALIGN (type) || TYPE_ALIGN (type) > 128))
    return true;
  if (TYPE_ALIGN (type) < 128)
    return false;

  if (AGGREGATE_TYPE_P (type))
    {
      /* Walk the aggregates recursively.  */
      switch (TREE_CODE (type))
	{
	case RECORD_TYPE:
	case UNION_TYPE:
	case QUAL_UNION_TYPE:
	  {
	    tree field;

	    /* Walk all the structure fields.  */
	    for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
	      {
		if (TREE_CODE (field) == FIELD_DECL
		    && ix86_compat_aligned_value_p (TREE_TYPE (field)))
		  return true;
	      }
	    break;
	  }

	case ARRAY_TYPE:
	  /* Just for use if some languages passes arrays by value.  */
	  if (ix86_compat_aligned_value_p (TREE_TYPE (type)))
	    return true;
	  break;

	default:
	  gcc_unreachable ();
	}
    }
  return false;
}

/* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
   XXX: This function is obsolete and is only used for checking psABI
   compatibility with previous versions of GCC.  */

static unsigned int
ix86_compat_function_arg_boundary (machine_mode mode,
				   const_tree type, unsigned int align)
{
  /* In 32bit, only _Decimal128 and __float128 are aligned to their
     natural boundaries.  */
  if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
    {
      /* i386 ABI defines all arguments to be 4 byte aligned.  We have to
	 make an exception for SSE modes since these require 128bit
	 alignment.

	 The handling here differs from field_alignment.  ICC aligns MMX
	 arguments to 4 byte boundaries, while structure fields are aligned
	 to 8 byte boundaries.  */
      if (!type)
	{
	  if (!(TARGET_SSE && SSE_REG_MODE_P (mode)))
	    align = PARM_BOUNDARY;
	}
      else
	{
	  if (!ix86_compat_aligned_value_p (type))
	    align = PARM_BOUNDARY;
	}
    }
  if (align > BIGGEST_ALIGNMENT)
    align = BIGGEST_ALIGNMENT;
  return align;
}

/* Return true when TYPE should be 128bit aligned for 32bit argument
   passing ABI.  */

static bool
ix86_contains_aligned_value_p (const_tree type)
{
  machine_mode mode = TYPE_MODE (type);

  if (mode == XFmode || mode == XCmode)
    return false;

  if (TYPE_ALIGN (type) < 128)
    return false;

  if (AGGREGATE_TYPE_P (type))
    {
      /* Walk the aggregates recursively.  */
      switch (TREE_CODE (type))
	{
	case RECORD_TYPE:
	case UNION_TYPE:
	case QUAL_UNION_TYPE:
	  {
	    tree field;

	    /* Walk all the structure fields.  */
	    for (field = TYPE_FIELDS (type);
		 field;
		 field = DECL_CHAIN (field))
	      {
		if (TREE_CODE (field) == FIELD_DECL
		    && ix86_contains_aligned_value_p (TREE_TYPE (field)))
		  return true;
	      }
	    break;
	  }

	case ARRAY_TYPE:
	  /* Just for use if some languages passes arrays by value.  */
	  if (ix86_contains_aligned_value_p (TREE_TYPE (type)))
	    return true;
	  break;

	default:
	  gcc_unreachable ();
	}
    }
  else
    return TYPE_ALIGN (type) >= 128;

  return false;
}

/* Gives the alignment boundary, in bits, of an argument with the
   specified mode and type.  */

static unsigned int
ix86_function_arg_boundary (machine_mode mode, const_tree type)
{
  unsigned int align;
  if (type)
    {
      /* Since the main variant type is used for call, we convert it to
	 the main variant type.  */
      type = TYPE_MAIN_VARIANT (type);
      align = TYPE_ALIGN (type);
    }
  else
    align = GET_MODE_ALIGNMENT (mode);
  if (align < PARM_BOUNDARY)
    align = PARM_BOUNDARY;
  else
    {
      static bool warned;
      unsigned int saved_align = align;

      if (!TARGET_64BIT)
	{
	  /* i386 ABI defines XFmode arguments to be 4 byte aligned.  */
	  if (!type)
	    {
	      if (mode == XFmode || mode == XCmode)
		align = PARM_BOUNDARY;
	    }
	  else if (!ix86_contains_aligned_value_p (type))
	    align = PARM_BOUNDARY;

	  if (align < 128)
	    align = PARM_BOUNDARY;
	}

      if (warn_psabi
	  && !warned
	  && align != ix86_compat_function_arg_boundary (mode, type,
							 saved_align))
	{
	  warned = true;
	  inform (input_location,
		  "The ABI for passing parameters with %d-byte"
		  " alignment has changed in GCC 4.6",
		  align / BITS_PER_UNIT);
	}
    }

  return align;
}

/* Return true if N is a possible register number of function value.  */

static bool
ix86_function_value_regno_p (const unsigned int regno)
{
  switch (regno)
    {
    case AX_REG:
      return true;
    case DX_REG:
      return (!TARGET_64BIT || ix86_cfun_abi () != MS_ABI);
    case DI_REG:
    case SI_REG:
      return TARGET_64BIT && ix86_cfun_abi () != MS_ABI;

    case BND0_REG:
    case BND1_REG:
      return chkp_function_instrumented_p (current_function_decl);

      /* Complex values are returned in %st(0)/%st(1) pair.  */
    case ST0_REG:
    case ST1_REG:
      /* TODO: The function should depend on current function ABI but
       builtins.c would need updating then. Therefore we use the
       default ABI.  */
      if (TARGET_64BIT && ix86_cfun_abi () == MS_ABI)
	return false;
      return TARGET_FLOAT_RETURNS_IN_80387;

      /* Complex values are returned in %xmm0/%xmm1 pair.  */
    case XMM0_REG:
    case XMM1_REG:
      return TARGET_SSE;

    case MM0_REG:
      if (TARGET_MACHO || TARGET_64BIT)
	return false;
      return TARGET_MMX;
    }

  return false;
}

/* Define how to find the value returned by a function.
   VALTYPE is the data type of the value (as a tree).
   If the precise function being called is known, FUNC is its FUNCTION_DECL;
   otherwise, FUNC is 0.  */

static rtx
function_value_32 (machine_mode orig_mode, machine_mode mode,
		   const_tree fntype, const_tree fn)
{
  unsigned int regno;

  /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
     we normally prevent this case when mmx is not available.  However
     some ABIs may require the result to be returned like DImode.  */
  if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
    regno = FIRST_MMX_REG;

  /* 16-byte vector modes in %xmm0.  See ix86_return_in_memory for where
     we prevent this case when sse is not available.  However some ABIs
     may require the result to be returned like integer TImode.  */
  else if (mode == TImode
	   || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
    regno = FIRST_SSE_REG;

  /* 32-byte vector modes in %ymm0.   */
  else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 32)
    regno = FIRST_SSE_REG;

  /* 64-byte vector modes in %zmm0.   */
  else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
    regno = FIRST_SSE_REG;

  /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387).  */
  else if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387)
    regno = FIRST_FLOAT_REG;
  else
    /* Most things go in %eax.  */
    regno = AX_REG;

  /* Override FP return register with %xmm0 for local functions when
     SSE math is enabled or for functions with sseregparm attribute.  */
  if ((fn || fntype) && (mode == SFmode || mode == DFmode))
    {
      int sse_level = ix86_function_sseregparm (fntype, fn, false);
      if (sse_level == -1)
	{
	  error ("calling %qD with SSE calling convention without "
		 "SSE/SSE2 enabled", fn);
	  sorry ("this is a GCC bug that can be worked around by adding "
		 "attribute used to function called");
	}
      else if ((sse_level >= 1 && mode == SFmode)
	       || (sse_level == 2 && mode == DFmode))
	regno = FIRST_SSE_REG;
    }

  /* OImode shouldn't be used directly.  */
  gcc_assert (mode != OImode);

  return gen_rtx_REG (orig_mode, regno);
}

static rtx
function_value_64 (machine_mode orig_mode, machine_mode mode,
		   const_tree valtype)
{
  rtx ret;

  /* Handle libcalls, which don't provide a type node.  */
  if (valtype == NULL)
    {
      unsigned int regno;

      switch (mode)
	{
	case SFmode:
	case SCmode:
	case DFmode:
	case DCmode:
	case TFmode:
	case SDmode:
	case DDmode:
	case TDmode:
	  regno = FIRST_SSE_REG;
	  break;
	case XFmode:
	case XCmode:
	  regno = FIRST_FLOAT_REG;
	  break;
	case TCmode:
	  return NULL;
	default:
	  regno = AX_REG;
	}

      return gen_rtx_REG (mode, regno);
    }
  else if (POINTER_TYPE_P (valtype))
    {
      /* Pointers are always returned in word_mode.  */
      mode = word_mode;
    }

  ret = construct_container (mode, orig_mode, valtype, 1,
			     X86_64_REGPARM_MAX, X86_64_SSE_REGPARM_MAX,
			     x86_64_int_return_registers, 0);

  /* For zero sized structures, construct_container returns NULL, but we
     need to keep rest of compiler happy by returning meaningful value.  */
  if (!ret)
    ret = gen_rtx_REG (orig_mode, AX_REG);

  return ret;
}

static rtx
function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
		      const_tree valtype)
{
  unsigned int regno = AX_REG;

  if (TARGET_SSE)
    {
      switch (GET_MODE_SIZE (mode))
	{
	case 16:
	  if (valtype != NULL_TREE
	      && !VECTOR_INTEGER_TYPE_P (valtype)
	      && !VECTOR_INTEGER_TYPE_P (valtype)
	      && !INTEGRAL_TYPE_P (valtype)
	      && !VECTOR_FLOAT_TYPE_P (valtype))
	    break;
	  if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
	      && !COMPLEX_MODE_P (mode))
	    regno = FIRST_SSE_REG;
	  break;
	case 8:
	case 4:
	  if (mode == SFmode || mode == DFmode)
	    regno = FIRST_SSE_REG;
	  break;
	default:
	  break;
        }
    }
  return gen_rtx_REG (orig_mode, regno);
}

static rtx
ix86_function_value_1 (const_tree valtype, const_tree fntype_or_decl,
		       machine_mode orig_mode, machine_mode mode)
{
  const_tree fn, fntype;

  fn = NULL_TREE;
  if (fntype_or_decl && DECL_P (fntype_or_decl))
    fn = fntype_or_decl;
  fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;

  if ((valtype && POINTER_BOUNDS_TYPE_P (valtype))
      || POINTER_BOUNDS_MODE_P (mode))
    return gen_rtx_REG (BNDmode, FIRST_BND_REG);
  else if (TARGET_64BIT && ix86_function_type_abi (fntype) == MS_ABI)
    return function_value_ms_64 (orig_mode, mode, valtype);
  else if (TARGET_64BIT)
    return function_value_64 (orig_mode, mode, valtype);
  else
    return function_value_32 (orig_mode, mode, fntype, fn);
}

static rtx
ix86_function_value (const_tree valtype, const_tree fntype_or_decl, bool)
{
  machine_mode mode, orig_mode;

  orig_mode = TYPE_MODE (valtype);
  mode = type_natural_mode (valtype, NULL, true);
  return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode);
}

/*  Return an RTX representing a place where a function returns
    or recieves pointer bounds or NULL if no bounds are returned.

    VALTYPE is a data type of a value returned by the function.

    FN_DECL_OR_TYPE is a tree node representing FUNCTION_DECL
    or FUNCTION_TYPE of the function.

    If OUTGOING is false, return a place in which the caller will
    see the return value.  Otherwise, return a place where a
    function returns a value.  */

static rtx
ix86_function_value_bounds (const_tree valtype,
			    const_tree fntype_or_decl ATTRIBUTE_UNUSED,
			    bool outgoing ATTRIBUTE_UNUSED)
{
  rtx res = NULL_RTX;

  if (BOUNDED_TYPE_P (valtype))
    res = gen_rtx_REG (BNDmode, FIRST_BND_REG);
  else if (chkp_type_has_pointer (valtype))
    {
      bitmap slots;
      rtx bounds[2];
      bitmap_iterator bi;
      unsigned i, bnd_no = 0;

      bitmap_obstack_initialize (NULL);
      slots = BITMAP_ALLOC (NULL);
      chkp_find_bound_slots (valtype, slots);

      EXECUTE_IF_SET_IN_BITMAP (slots, 0, i, bi)
	{
	  rtx reg = gen_rtx_REG (BNDmode, FIRST_BND_REG + bnd_no);
	  rtx offs = GEN_INT (i * POINTER_SIZE / BITS_PER_UNIT);
	  gcc_assert (bnd_no < 2);
	  bounds[bnd_no++] = gen_rtx_EXPR_LIST (VOIDmode, reg, offs);
	}

      res = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (bnd_no, bounds));

      BITMAP_FREE (slots);
      bitmap_obstack_release (NULL);
    }
  else
    res = NULL_RTX;

  return res;
}

/* Pointer function arguments and return values are promoted to
   word_mode for normal functions.  */

static machine_mode
ix86_promote_function_mode (const_tree type, machine_mode mode,
			    int *punsignedp, const_tree fntype,
			    int for_return)
{
  if (cfun->machine->func_type == TYPE_NORMAL
      && type != NULL_TREE
      && POINTER_TYPE_P (type))
    {
      *punsignedp = POINTERS_EXTEND_UNSIGNED;
      return word_mode;
    }
  return default_promote_function_mode (type, mode, punsignedp, fntype,
					for_return);
}

/* Return true if a structure, union or array with MODE containing FIELD
   should be accessed using BLKmode.  */

static bool
ix86_member_type_forces_blk (const_tree field, machine_mode mode)
{
  /* Union with XFmode must be in BLKmode.  */
  return (mode == XFmode
	  && (TREE_CODE (DECL_FIELD_CONTEXT (field)) == UNION_TYPE
	      || TREE_CODE (DECL_FIELD_CONTEXT (field)) == QUAL_UNION_TYPE));
}

rtx
ix86_libcall_value (machine_mode mode)
{
  return ix86_function_value_1 (NULL, NULL, mode, mode);
}

/* Return true iff type is returned in memory.  */

static bool
ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
{
#ifdef SUBTARGET_RETURN_IN_MEMORY
  return SUBTARGET_RETURN_IN_MEMORY (type, fntype);
#else
  const machine_mode mode = type_natural_mode (type, NULL, true);
  HOST_WIDE_INT size;

  if (POINTER_BOUNDS_TYPE_P (type))
    return false;

  if (TARGET_64BIT)
    {
      if (ix86_function_type_abi (fntype) == MS_ABI)
	{
	  size = int_size_in_bytes (type);

	  /* __m128 is returned in xmm0.  */
	  if ((!type || VECTOR_INTEGER_TYPE_P (type)
	       || INTEGRAL_TYPE_P (type)
	       || VECTOR_FLOAT_TYPE_P (type))
	      && (SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
	      && !COMPLEX_MODE_P (mode)
	      && (GET_MODE_SIZE (mode) == 16 || size == 16))
	    return false;

	  /* Otherwise, the size must be exactly in [1248]. */
	  return size != 1 && size != 2 && size != 4 && size != 8;
	}
      else
	{
	  int needed_intregs, needed_sseregs;

	  return examine_argument (mode, type, 1,
				   &needed_intregs, &needed_sseregs);
	}
    }
  else
    {
      size = int_size_in_bytes (type);

      /* Intel MCU psABI returns scalars and aggregates no larger than 8
	 bytes in registers.  */
      if (TARGET_IAMCU)
	return VECTOR_MODE_P (mode) || size < 0 || size > 8;

      if (mode == BLKmode)
	return true;

      if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8)
	return false;

      if (VECTOR_MODE_P (mode) || mode == TImode)
	{
	  /* User-created vectors small enough to fit in EAX.  */
	  if (size < 8)
	    return false;

	  /* Unless ABI prescibes otherwise,
	     MMX/3dNow values are returned in MM0 if available.  */
	     
	  if (size == 8)
	    return TARGET_VECT8_RETURNS || !TARGET_MMX;

	  /* SSE values are returned in XMM0 if available.  */
	  if (size == 16)
	    return !TARGET_SSE;

	  /* AVX values are returned in YMM0 if available.  */
	  if (size == 32)
	    return !TARGET_AVX;

	  /* AVX512F values are returned in ZMM0 if available.  */
	  if (size == 64)
	    return !TARGET_AVX512F;
	}

      if (mode == XFmode)
	return false;

      if (size > 12)
	return true;

      /* OImode shouldn't be used directly.  */
      gcc_assert (mode != OImode);

      return false;
    }
#endif
}


/* Create the va_list data type.  */

static tree
ix86_build_builtin_va_list_64 (void)
{
  tree f_gpr, f_fpr, f_ovf, f_sav, record, type_decl;

  record = lang_hooks.types.make_type (RECORD_TYPE);
  type_decl = build_decl (BUILTINS_LOCATION,
			  TYPE_DECL, get_identifier ("__va_list_tag"), record);

  f_gpr = build_decl (BUILTINS_LOCATION,
		      FIELD_DECL, get_identifier ("gp_offset"),
		      unsigned_type_node);
  f_fpr = build_decl (BUILTINS_LOCATION,
		      FIELD_DECL, get_identifier ("fp_offset"),
		      unsigned_type_node);
  f_ovf = build_decl (BUILTINS_LOCATION,
		      FIELD_DECL, get_identifier ("overflow_arg_area"),
		      ptr_type_node);
  f_sav = build_decl (BUILTINS_LOCATION,
		      FIELD_DECL, get_identifier ("reg_save_area"),
		      ptr_type_node);

  va_list_gpr_counter_field = f_gpr;
  va_list_fpr_counter_field = f_fpr;

  DECL_FIELD_CONTEXT (f_gpr) = record;
  DECL_FIELD_CONTEXT (f_fpr) = record;
  DECL_FIELD_CONTEXT (f_ovf) = record;
  DECL_FIELD_CONTEXT (f_sav) = record;

  TYPE_STUB_DECL (record) = type_decl;
  TYPE_NAME (record) = type_decl;
  TYPE_FIELDS (record) = f_gpr;
  DECL_CHAIN (f_gpr) = f_fpr;
  DECL_CHAIN (f_fpr) = f_ovf;
  DECL_CHAIN (f_ovf) = f_sav;

  layout_type (record);

  TYPE_ATTRIBUTES (record) = tree_cons (get_identifier ("sysv_abi va_list"),
					NULL_TREE, TYPE_ATTRIBUTES (record));

  /* The correct type is an array type of one element.  */
  return build_array_type (record, build_index_type (size_zero_node));
}

/* Setup the builtin va_list data type and for 64-bit the additional
   calling convention specific va_list data types.  */

static tree
ix86_build_builtin_va_list (void)
{
  if (TARGET_64BIT)
    {
      /* Initialize ABI specific va_list builtin types.

	 In lto1, we can encounter two va_list types:
	 - one as a result of the type-merge across TUs, and
	 - the one constructed here.
	 These two types will not have the same TYPE_MAIN_VARIANT, and therefore
	 a type identity check in canonical_va_list_type based on
	 TYPE_MAIN_VARIANT (which we used to have) will not work.
	 Instead, we tag each va_list_type_node with its unique attribute, and
	 look for the attribute in the type identity check in
	 canonical_va_list_type.

	 Tagging sysv_va_list_type_node directly with the attribute is
	 problematic since it's a array of one record, which will degrade into a
	 pointer to record when used as parameter (see build_va_arg comments for
	 an example), dropping the attribute in the process.  So we tag the
	 record instead.  */

      /* For SYSV_ABI we use an array of one record.  */
      sysv_va_list_type_node = ix86_build_builtin_va_list_64 ();
	
      /* For MS_ABI we use plain pointer to argument area.  */
      tree char_ptr_type = build_pointer_type (char_type_node);
      tree attr = tree_cons (get_identifier ("ms_abi va_list"), NULL_TREE,
			     TYPE_ATTRIBUTES (char_ptr_type));
      ms_va_list_type_node = build_type_attribute_variant (char_ptr_type, attr);

      return ((ix86_abi == MS_ABI)
	      ? ms_va_list_type_node
	      : sysv_va_list_type_node);
    }
  else
    {
      /* For i386 we use plain pointer to argument area.  */
      return build_pointer_type (char_type_node);
    }
}

/* Worker function for TARGET_SETUP_INCOMING_VARARGS.  */

static void
setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum)
{
  rtx save_area, mem;
  alias_set_type set;
  int i, max;

  /* GPR size of varargs save area.  */
  if (cfun->va_list_gpr_size)
    ix86_varargs_gpr_size = X86_64_REGPARM_MAX * UNITS_PER_WORD;
  else
    ix86_varargs_gpr_size = 0;

  /* FPR size of varargs save area.  We don't need it if we don't pass
     anything in SSE registers.  */
  if (TARGET_SSE && cfun->va_list_fpr_size)
    ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16;
  else
    ix86_varargs_fpr_size = 0;

  if (! ix86_varargs_gpr_size && ! ix86_varargs_fpr_size)
    return;

  save_area = frame_pointer_rtx;
  set = get_varargs_alias_set ();

  max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
  if (max > X86_64_REGPARM_MAX)
    max = X86_64_REGPARM_MAX;

  for (i = cum->regno; i < max; i++)
    {
      mem = gen_rtx_MEM (word_mode,
			 plus_constant (Pmode, save_area, i * UNITS_PER_WORD));
      MEM_NOTRAP_P (mem) = 1;
      set_mem_alias_set (mem, set);
      emit_move_insn (mem,
		      gen_rtx_REG (word_mode,
				   x86_64_int_parameter_registers[i]));
    }

  if (ix86_varargs_fpr_size)
    {
      machine_mode smode;
      rtx_code_label *label;
      rtx test;

      /* Now emit code to save SSE registers.  The AX parameter contains number
	 of SSE parameter registers used to call this function, though all we
	 actually check here is the zero/non-zero status.  */

      label = gen_label_rtx ();
      test = gen_rtx_EQ (VOIDmode, gen_rtx_REG (QImode, AX_REG), const0_rtx);
      emit_jump_insn (gen_cbranchqi4 (test, XEXP (test, 0), XEXP (test, 1),
				      label));

      /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
	 we used movdqa (i.e. TImode) instead?  Perhaps even better would
	 be if we could determine the real mode of the data, via a hook
	 into pass_stdarg.  Ignore all that for now.  */
      smode = V4SFmode;
      if (crtl->stack_alignment_needed < GET_MODE_ALIGNMENT (smode))
	crtl->stack_alignment_needed = GET_MODE_ALIGNMENT (smode);

      max = cum->sse_regno + cfun->va_list_fpr_size / 16;
      if (max > X86_64_SSE_REGPARM_MAX)
	max = X86_64_SSE_REGPARM_MAX;

      for (i = cum->sse_regno; i < max; ++i)
	{
	  mem = plus_constant (Pmode, save_area,
			       i * 16 + ix86_varargs_gpr_size);
	  mem = gen_rtx_MEM (smode, mem);
	  MEM_NOTRAP_P (mem) = 1;
	  set_mem_alias_set (mem, set);
	  set_mem_align (mem, GET_MODE_ALIGNMENT (smode));

	  emit_move_insn (mem, gen_rtx_REG (smode, SSE_REGNO (i)));
	}

      emit_label (label);
    }
}

static void
setup_incoming_varargs_ms_64 (CUMULATIVE_ARGS *cum)
{
  alias_set_type set = get_varargs_alias_set ();
  int i;

  /* Reset to zero, as there might be a sysv vaarg used
     before.  */
  ix86_varargs_gpr_size = 0;
  ix86_varargs_fpr_size = 0;

  for (i = cum->regno; i < X86_64_MS_REGPARM_MAX; i++)
    {
      rtx reg, mem;

      mem = gen_rtx_MEM (Pmode,
			 plus_constant (Pmode, virtual_incoming_args_rtx,
					i * UNITS_PER_WORD));
      MEM_NOTRAP_P (mem) = 1;
      set_mem_alias_set (mem, set);

      reg = gen_rtx_REG (Pmode, x86_64_ms_abi_int_parameter_registers[i]);
      emit_move_insn (mem, reg);
    }
}

static void
ix86_setup_incoming_varargs (cumulative_args_t cum_v, machine_mode mode,
			     tree type, int *, int no_rtl)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
  CUMULATIVE_ARGS next_cum;
  tree fntype;

  /* This argument doesn't appear to be used anymore.  Which is good,
     because the old code here didn't suppress rtl generation.  */
  gcc_assert (!no_rtl);

  if (!TARGET_64BIT)
    return;

  fntype = TREE_TYPE (current_function_decl);

  /* For varargs, we do not want to skip the dummy va_dcl argument.
     For stdargs, we do want to skip the last named argument.  */
  next_cum = *cum;
  if (stdarg_p (fntype))
    ix86_function_arg_advance (pack_cumulative_args (&next_cum), mode, type,
			       true);

  if (cum->call_abi == MS_ABI)
    setup_incoming_varargs_ms_64 (&next_cum);
  else
    setup_incoming_varargs_64 (&next_cum);
}

static void
ix86_setup_incoming_vararg_bounds (cumulative_args_t cum_v,
				   enum machine_mode mode,
				   tree type,
				   int *pretend_size ATTRIBUTE_UNUSED,
				   int no_rtl)
{
  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
  CUMULATIVE_ARGS next_cum;
  tree fntype;
  rtx save_area;
  int bnd_reg, i, max;

  gcc_assert (!no_rtl);

  /* Do nothing if we use plain pointer to argument area.  */
  if (!TARGET_64BIT || cum->call_abi == MS_ABI)
    return;

  fntype = TREE_TYPE (current_function_decl);

  /* For varargs, we do not want to skip the dummy va_dcl argument.
     For stdargs, we do want to skip the last named argument.  */
  next_cum = *cum;
  if (stdarg_p (fntype))
    ix86_function_arg_advance (pack_cumulative_args (&next_cum), mode, type,
			       true);
  save_area = frame_pointer_rtx;

  max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
  if (max > X86_64_REGPARM_MAX)
    max = X86_64_REGPARM_MAX;

  bnd_reg = cum->bnd_regno + cum->force_bnd_pass;
  if (chkp_function_instrumented_p (current_function_decl))
    for (i = cum->regno; i < max; i++)
      {
	rtx addr = plus_constant (Pmode, save_area, i * UNITS_PER_WORD);
	rtx ptr = gen_rtx_REG (Pmode,
			       x86_64_int_parameter_registers[i]);
	rtx bounds;

	if (bnd_reg <= LAST_BND_REG)
	  bounds = gen_rtx_REG (BNDmode, bnd_reg);
	else
	  {
	    rtx ldx_addr =
	      plus_constant (Pmode, arg_pointer_rtx,
			     (LAST_BND_REG - bnd_reg) * GET_MODE_SIZE (Pmode));
	    bounds = gen_reg_rtx (BNDmode);
	    emit_insn (BNDmode == BND64mode
		       ? gen_bnd64_ldx (bounds, ldx_addr, ptr)
		       : gen_bnd32_ldx (bounds, ldx_addr, ptr));
	  }

	emit_insn (BNDmode == BND64mode
		   ? gen_bnd64_stx (addr, ptr, bounds)
		   : gen_bnd32_stx (addr, ptr, bounds));

	bnd_reg++;
      }
}


/* Checks if TYPE is of kind va_list char *.  */

static bool
is_va_list_char_pointer (tree type)
{
  tree canonic;

  /* For 32-bit it is always true.  */
  if (!TARGET_64BIT)
    return true;
  canonic = ix86_canonical_va_list_type (type);
  return (canonic == ms_va_list_type_node
          || (ix86_abi == MS_ABI && canonic == va_list_type_node));
}

/* Implement va_start.  */

static void
ix86_va_start (tree valist, rtx nextarg)
{
  HOST_WIDE_INT words, n_gpr, n_fpr;
  tree f_gpr, f_fpr, f_ovf, f_sav;
  tree gpr, fpr, ovf, sav, t;
  tree type;
  rtx ovf_rtx;

  if (flag_split_stack
      && cfun->machine->split_stack_varargs_pointer == NULL_RTX)
    {
      unsigned int scratch_regno;

      /* When we are splitting the stack, we can't refer to the stack
	 arguments using internal_arg_pointer, because they may be on
	 the old stack.  The split stack prologue will arrange to
	 leave a pointer to the old stack arguments in a scratch
	 register, which we here copy to a pseudo-register.  The split
	 stack prologue can't set the pseudo-register directly because
	 it (the prologue) runs before any registers have been saved.  */

      scratch_regno = split_stack_prologue_scratch_regno ();
      if (scratch_regno != INVALID_REGNUM)
	{
	  rtx reg;
	  rtx_insn *seq;

	  reg = gen_reg_rtx (Pmode);
	  cfun->machine->split_stack_varargs_pointer = reg;

	  start_sequence ();
	  emit_move_insn (reg, gen_rtx_REG (Pmode, scratch_regno));
	  seq = get_insns ();
	  end_sequence ();

	  push_topmost_sequence ();
	  emit_insn_after (seq, entry_of_function ());
	  pop_topmost_sequence ();
	}
    }

  /* Only 64bit target needs something special.  */
  if (is_va_list_char_pointer (TREE_TYPE (valist)))
    {
      if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
	std_expand_builtin_va_start (valist, nextarg);
      else
	{
	  rtx va_r, next;

	  va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE);
	  next = expand_binop (ptr_mode, add_optab,
			       cfun->machine->split_stack_varargs_pointer,
			       crtl->args.arg_offset_rtx,
			       NULL_RTX, 0, OPTAB_LIB_WIDEN);
	  convert_move (va_r, next, 0);

	  /* Store zero bounds for va_list.  */
	  if (chkp_function_instrumented_p (current_function_decl))
	    chkp_expand_bounds_reset_for_mem (valist,
					      make_tree (TREE_TYPE (valist),
							 next));

	}
      return;
    }

  f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
  f_fpr = DECL_CHAIN (f_gpr);
  f_ovf = DECL_CHAIN (f_fpr);
  f_sav = DECL_CHAIN (f_ovf);

  valist = build_simple_mem_ref (valist);
  TREE_TYPE (valist) = TREE_TYPE (sysv_va_list_type_node);
  /* The following should be folded into the MEM_REF offset.  */
  gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), unshare_expr (valist),
		f_gpr, NULL_TREE);
  fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), unshare_expr (valist),
		f_fpr, NULL_TREE);
  ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), unshare_expr (valist),
		f_ovf, NULL_TREE);
  sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), unshare_expr (valist),
		f_sav, NULL_TREE);

  /* Count number of gp and fp argument registers used.  */
  words = crtl->args.info.words;
  n_gpr = crtl->args.info.regno;
  n_fpr = crtl->args.info.sse_regno;

  if (cfun->va_list_gpr_size)
    {
      type = TREE_TYPE (gpr);
      t = build2 (MODIFY_EXPR, type,
		  gpr, build_int_cst (type, n_gpr * 8));
      TREE_SIDE_EFFECTS (t) = 1;
      expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
    }

  if (TARGET_SSE && cfun->va_list_fpr_size)
    {
      type = TREE_TYPE (fpr);
      t = build2 (MODIFY_EXPR, type, fpr,
		  build_int_cst (type, n_fpr * 16 + 8*X86_64_REGPARM_MAX));
      TREE_SIDE_EFFECTS (t) = 1;
      expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
    }

  /* Find the overflow area.  */
  type = TREE_TYPE (ovf);
  if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
    ovf_rtx = crtl->args.internal_arg_pointer;
  else
    ovf_rtx = cfun->machine->split_stack_varargs_pointer;
  t = make_tree (type, ovf_rtx);
  if (words != 0)
    t = fold_build_pointer_plus_hwi (t, words * UNITS_PER_WORD);

  /* Store zero bounds for overflow area pointer.  */
  if (chkp_function_instrumented_p (current_function_decl))
    chkp_expand_bounds_reset_for_mem (ovf, t);

  t = build2 (MODIFY_EXPR, type, ovf, t);
  TREE_SIDE_EFFECTS (t) = 1;
  expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);

  if (ix86_varargs_gpr_size || ix86_varargs_fpr_size)
    {
      /* Find the register save area.
	 Prologue of the function save it right above stack frame.  */
      type = TREE_TYPE (sav);
      t = make_tree (type, frame_pointer_rtx);
      if (!ix86_varargs_gpr_size)
	t = fold_build_pointer_plus_hwi (t, -8 * X86_64_REGPARM_MAX);

      /* Store zero bounds for save area pointer.  */
      if (chkp_function_instrumented_p (current_function_decl))
	chkp_expand_bounds_reset_for_mem (sav, t);

      t = build2 (MODIFY_EXPR, type, sav, t);
      TREE_SIDE_EFFECTS (t) = 1;
      expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
    }
}

/* Implement va_arg.  */

static tree
ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
		      gimple_seq *post_p)
{
  static const int intreg[6] = { 0, 1, 2, 3, 4, 5 };
  tree f_gpr, f_fpr, f_ovf, f_sav;
  tree gpr, fpr, ovf, sav, t;
  int size, rsize;
  tree lab_false, lab_over = NULL_TREE;
  tree addr, t2;
  rtx container;
  int indirect_p = 0;
  tree ptrtype;
  machine_mode nat_mode;
  unsigned int arg_boundary;

  /* Only 64bit target needs something special.  */
  if (is_va_list_char_pointer (TREE_TYPE (valist)))
    return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);

  f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
  f_fpr = DECL_CHAIN (f_gpr);
  f_ovf = DECL_CHAIN (f_fpr);
  f_sav = DECL_CHAIN (f_ovf);

  gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr),
		valist, f_gpr, NULL_TREE);

  fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
  ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
  sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);

  indirect_p = pass_by_reference (NULL, TYPE_MODE (type), type, false);
  if (indirect_p)
    type = build_pointer_type (type);
  size = int_size_in_bytes (type);
  rsize = CEIL (size, UNITS_PER_WORD);

  nat_mode = type_natural_mode (type, NULL, false);
  switch (nat_mode)
    {
    case V8SFmode:
    case V8SImode:
    case V32QImode:
    case V16HImode:
    case V4DFmode:
    case V4DImode:
    case V16SFmode:
    case V16SImode:
    case V64QImode:
    case V32HImode:
    case V8DFmode:
    case V8DImode:
      /* Unnamed 256 and 512bit vector mode parameters are passed on stack.  */
      if (!TARGET_64BIT_MS_ABI)
	{
	  container = NULL;
	  break;
	}
      /* FALLTHRU */

    default:
      container = construct_container (nat_mode, TYPE_MODE (type),
				       type, 0, X86_64_REGPARM_MAX,
				       X86_64_SSE_REGPARM_MAX, intreg,
				       0);
      break;
    }

  /* Pull the value out of the saved registers.  */

  addr = create_tmp_var (ptr_type_node, "addr");

  if (container)
    {
      int needed_intregs, needed_sseregs;
      bool need_temp;
      tree int_addr, sse_addr;

      lab_false = create_artificial_label (UNKNOWN_LOCATION);
      lab_over = create_artificial_label (UNKNOWN_LOCATION);

      examine_argument (nat_mode, type, 0, &needed_intregs, &needed_sseregs);

      need_temp = (!REG_P (container)
		   && ((needed_intregs && TYPE_ALIGN (type) > 64)
		       || TYPE_ALIGN (type) > 128));

      /* In case we are passing structure, verify that it is consecutive block
         on the register save area.  If not we need to do moves.  */
      if (!need_temp && !REG_P (container))
	{
	  /* Verify that all registers are strictly consecutive  */
	  if (SSE_REGNO_P (REGNO (XEXP (XVECEXP (container, 0, 0), 0))))
	    {
	      int i;

	      for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
		{
		  rtx slot = XVECEXP (container, 0, i);
		  if (REGNO (XEXP (slot, 0)) != FIRST_SSE_REG + (unsigned int) i
		      || INTVAL (XEXP (slot, 1)) != i * 16)
		    need_temp = true;
		}
	    }
	  else
	    {
	      int i;

	      for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
		{
		  rtx slot = XVECEXP (container, 0, i);
		  if (REGNO (XEXP (slot, 0)) != (unsigned int) i
		      || INTVAL (XEXP (slot, 1)) != i * 8)
		    need_temp = true;
		}
	    }
	}
      if (!need_temp)
	{
	  int_addr = addr;
	  sse_addr = addr;
	}
      else
	{
	  int_addr = create_tmp_var (ptr_type_node, "int_addr");
	  sse_addr = create_tmp_var (ptr_type_node, "sse_addr");
	}

      /* First ensure that we fit completely in registers.  */
      if (needed_intregs)
	{
	  t = build_int_cst (TREE_TYPE (gpr),
			     (X86_64_REGPARM_MAX - needed_intregs + 1) * 8);
	  t = build2 (GE_EXPR, boolean_type_node, gpr, t);
	  t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
	  t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
	  gimplify_and_add (t, pre_p);
	}
      if (needed_sseregs)
	{
	  t = build_int_cst (TREE_TYPE (fpr),
			     (X86_64_SSE_REGPARM_MAX - needed_sseregs + 1) * 16
			     + X86_64_REGPARM_MAX * 8);
	  t = build2 (GE_EXPR, boolean_type_node, fpr, t);
	  t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
	  t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
	  gimplify_and_add (t, pre_p);
	}

      /* Compute index to start of area used for integer regs.  */
      if (needed_intregs)
	{
	  /* int_addr = gpr + sav; */
	  t = fold_build_pointer_plus (sav, gpr);
	  gimplify_assign (int_addr, t, pre_p);
	}
      if (needed_sseregs)
	{
	  /* sse_addr = fpr + sav; */
	  t = fold_build_pointer_plus (sav, fpr);
	  gimplify_assign (sse_addr, t, pre_p);
	}
      if (need_temp)
	{
	  int i, prev_size = 0;
	  tree temp = create_tmp_var (type, "va_arg_tmp");

	  /* addr = &temp; */
	  t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
	  gimplify_assign (addr, t, pre_p);

	  for (i = 0; i < XVECLEN (container, 0); i++)
	    {
	      rtx slot = XVECEXP (container, 0, i);
	      rtx reg = XEXP (slot, 0);
	      machine_mode mode = GET_MODE (reg);
	      tree piece_type;
	      tree addr_type;
	      tree daddr_type;
	      tree src_addr, src;
	      int src_offset;
	      tree dest_addr, dest;
	      int cur_size = GET_MODE_SIZE (mode);

	      gcc_assert (prev_size <= INTVAL (XEXP (slot, 1)));
	      prev_size = INTVAL (XEXP (slot, 1));
	      if (prev_size + cur_size > size)
		{
		  cur_size = size - prev_size;
		  mode = mode_for_size (cur_size * BITS_PER_UNIT, MODE_INT, 1);
		  if (mode == BLKmode)
		    mode = QImode;
		}
	      piece_type = lang_hooks.types.type_for_mode (mode, 1);
	      if (mode == GET_MODE (reg))
		addr_type = build_pointer_type (piece_type);
	      else
		addr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
							 true);
	      daddr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
							true);

	      if (SSE_REGNO_P (REGNO (reg)))
		{
		  src_addr = sse_addr;
		  src_offset = (REGNO (reg) - FIRST_SSE_REG) * 16;
		}
	      else
		{
		  src_addr = int_addr;
		  src_offset = REGNO (reg) * 8;
		}
	      src_addr = fold_convert (addr_type, src_addr);
	      src_addr = fold_build_pointer_plus_hwi (src_addr, src_offset);

	      dest_addr = fold_convert (daddr_type, addr);
	      dest_addr = fold_build_pointer_plus_hwi (dest_addr, prev_size);
	      if (cur_size == GET_MODE_SIZE (mode))
		{
		  src = build_va_arg_indirect_ref (src_addr);
		  dest = build_va_arg_indirect_ref (dest_addr);

		  gimplify_assign (dest, src, pre_p);
		}
	      else
		{
		  tree copy
		    = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMCPY),
				       3, dest_addr, src_addr,
				       size_int (cur_size));
		  gimplify_and_add (copy, pre_p);
		}
	      prev_size += cur_size;
	    }
	}

      if (needed_intregs)
	{
	  t = build2 (PLUS_EXPR, TREE_TYPE (gpr), gpr,
		      build_int_cst (TREE_TYPE (gpr), needed_intregs * 8));
	  gimplify_assign (gpr, t, pre_p);
	}

      if (needed_sseregs)
	{
	  t = build2 (PLUS_EXPR, TREE_TYPE (fpr), fpr,
		      build_int_cst (TREE_TYPE (fpr), needed_sseregs * 16));
	  gimplify_assign (unshare_expr (fpr), t, pre_p);
	}

      gimple_seq_add_stmt (pre_p, gimple_build_goto (lab_over));

      gimple_seq_add_stmt (pre_p, gimple_build_label (lab_false));
    }

  /* ... otherwise out of the overflow area.  */

  /* When we align parameter on stack for caller, if the parameter
     alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
     aligned at MAX_SUPPORTED_STACK_ALIGNMENT.  We will match callee
     here with caller.  */
  arg_boundary = ix86_function_arg_boundary (VOIDmode, type);
  if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
    arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT;

  /* Care for on-stack alignment if needed.  */
  if (arg_boundary <= 64 || size == 0)
    t = ovf;
 else
    {
      HOST_WIDE_INT align = arg_boundary / 8;
      t = fold_build_pointer_plus_hwi (ovf, align - 1);
      t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
		  build_int_cst (TREE_TYPE (t), -align));
    }

  gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue);
  gimplify_assign (addr, t, pre_p);

  t = fold_build_pointer_plus_hwi (t, rsize * UNITS_PER_WORD);
  gimplify_assign (unshare_expr (ovf), t, pre_p);

  if (container)
    gimple_seq_add_stmt (pre_p, gimple_build_label (lab_over));

  ptrtype = build_pointer_type_for_mode (type, ptr_mode, true);
  addr = fold_convert (ptrtype, addr);

  if (indirect_p)
    addr = build_va_arg_indirect_ref (addr);
  return build_va_arg_indirect_ref (addr);
}

/* Return true if OPNUM's MEM should be matched
   in movabs* patterns.  */

bool
ix86_check_movabs (rtx insn, int opnum)
{
  rtx set, mem;

  set = PATTERN (insn);
  if (GET_CODE (set) == PARALLEL)
    set = XVECEXP (set, 0, 0);
  gcc_assert (GET_CODE (set) == SET);
  mem = XEXP (set, opnum);
  while (SUBREG_P (mem))
    mem = SUBREG_REG (mem);
  gcc_assert (MEM_P (mem));
  return volatile_ok || !MEM_VOLATILE_P (mem);
}

/* Return false if INSN contains a MEM with a non-default address space.  */
bool
ix86_check_no_addr_space (rtx insn)
{
  subrtx_var_iterator::array_type array;
  FOR_EACH_SUBRTX_VAR (iter, array, PATTERN (insn), ALL)
    {
      rtx x = *iter;
      if (MEM_P (x) && !ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x)))
	return false;
    }
  return true;
}

/* Initialize the table of extra 80387 mathematical constants.  */

static void
init_ext_80387_constants (void)
{
  static const char * cst[5] =
  {
    "0.3010299956639811952256464283594894482",  /* 0: fldlg2  */
    "0.6931471805599453094286904741849753009",  /* 1: fldln2  */
    "1.4426950408889634073876517827983434472",  /* 2: fldl2e  */
    "3.3219280948873623478083405569094566090",  /* 3: fldl2t  */
    "3.1415926535897932385128089594061862044",  /* 4: fldpi   */
  };
  int i;

  for (i = 0; i < 5; i++)
    {
      real_from_string (&ext_80387_constants_table[i], cst[i]);
      /* Ensure each constant is rounded to XFmode precision.  */
      real_convert (&ext_80387_constants_table[i],
		    XFmode, &ext_80387_constants_table[i]);
    }

  ext_80387_constants_init = 1;
}

/* Return non-zero if the constant is something that
   can be loaded with a special instruction.  */

int
standard_80387_constant_p (rtx x)
{
  machine_mode mode = GET_MODE (x);

  const REAL_VALUE_TYPE *r;

  if (!(CONST_DOUBLE_P (x) && X87_FLOAT_MODE_P (mode)))
    return -1;

  if (x == CONST0_RTX (mode))
    return 1;
  if (x == CONST1_RTX (mode))
    return 2;

  r = CONST_DOUBLE_REAL_VALUE (x);

  /* For XFmode constants, try to find a special 80387 instruction when
     optimizing for size or on those CPUs that benefit from them.  */
  if (mode == XFmode
      && (optimize_function_for_size_p (cfun) || TARGET_EXT_80387_CONSTANTS))
    {
      int i;

      if (! ext_80387_constants_init)
	init_ext_80387_constants ();

      for (i = 0; i < 5; i++)
        if (real_identical (r, &ext_80387_constants_table[i]))
	  return i + 3;
    }

  /* Load of the constant -0.0 or -1.0 will be split as
     fldz;fchs or fld1;fchs sequence.  */
  if (real_isnegzero (r))
    return 8;
  if (real_identical (r, &dconstm1))
    return 9;

  return 0;
}

/* Return the opcode of the special instruction to be used to load
   the constant X.  */

const char *
standard_80387_constant_opcode (rtx x)
{
  switch (standard_80387_constant_p (x))
    {
    case 1:
      return "fldz";
    case 2:
      return "fld1";
    case 3:
      return "fldlg2";
    case 4:
      return "fldln2";
    case 5:
      return "fldl2e";
    case 6:
      return "fldl2t";
    case 7:
      return "fldpi";
    case 8:
    case 9:
      return "#";
    default:
      gcc_unreachable ();
    }
}

/* Return the CONST_DOUBLE representing the 80387 constant that is
   loaded by the specified special instruction.  The argument IDX
   matches the return value from standard_80387_constant_p.  */

rtx
standard_80387_constant_rtx (int idx)
{
  int i;

  if (! ext_80387_constants_init)
    init_ext_80387_constants ();

  switch (idx)
    {
    case 3:
    case 4:
    case 5:
    case 6:
    case 7:
      i = idx - 3;
      break;

    default:
      gcc_unreachable ();
    }

  return const_double_from_real_value (ext_80387_constants_table[i],
				       XFmode);
}

/* Return 1 if X is all bits 0 and 2 if X is all bits 1
   in supported SSE/AVX vector mode.  */

int
standard_sse_constant_p (rtx x, machine_mode pred_mode)
{
  machine_mode mode;

  if (!TARGET_SSE)
    return 0;

  mode = GET_MODE (x);

  if (x == const0_rtx || const0_operand (x, mode))
    return 1;

  if (x == constm1_rtx || vector_all_ones_operand (x, mode))
    {
      /* VOIDmode integer constant, get mode from the predicate.  */
      if (mode == VOIDmode)
	mode = pred_mode;

      switch (GET_MODE_SIZE (mode))
	{
	case 64:
	  if (TARGET_AVX512F)
	    return 2;
	  break;
	case 32:
	  if (TARGET_AVX2)
	    return 2;
	  break;
	case 16:
	  if (TARGET_SSE2)
	    return 2;
	  break;
	case 0:
	  /* VOIDmode */
	  gcc_unreachable ();
	default:
	  break;
	}
    }

  return 0;
}

/* Return the opcode of the special instruction to be used to load
   the constant X.  */

const char *
standard_sse_constant_opcode (rtx_insn *insn, rtx x)
{
  machine_mode mode;

  gcc_assert (TARGET_SSE);

  mode = GET_MODE (x);

  if (x == const0_rtx || const0_operand (x, mode))
    {
      switch (get_attr_mode (insn))
	{
	case MODE_XI:
	  return "vpxord\t%g0, %g0, %g0";
	case MODE_OI:
	  return (TARGET_AVX512VL
		  ? "vpxord\t%x0, %x0, %x0"
		  : "vpxor\t%x0, %x0, %x0");
	case MODE_TI:
	  return (TARGET_AVX512VL
		  ? "vpxord\t%t0, %t0, %t0"
		  : "%vpxor\t%0, %d0");

	case MODE_V8DF:
	  return (TARGET_AVX512DQ
		  ? "vxorpd\t%g0, %g0, %g0"
		  : "vpxorq\t%g0, %g0, %g0");
	case MODE_V4DF:
	  return "vxorpd\t%x0, %x0, %x0";
	case MODE_V2DF:
	  return "%vxorpd\t%0, %d0";

	case MODE_V16SF:
	  return (TARGET_AVX512DQ
		  ? "vxorps\t%g0, %g0, %g0"
		  : "vpxord\t%g0, %g0, %g0");
	case MODE_V8SF:
	  return "vxorps\t%x0, %x0, %x0";
	case MODE_V4SF:
	  return "%vxorps\t%0, %d0";

	default:
	  gcc_unreachable ();
	}
    }
  else if (x == constm1_rtx || vector_all_ones_operand (x, mode))
    {
      enum attr_mode insn_mode = get_attr_mode (insn);
      
      switch (insn_mode)
	{
	case MODE_XI:
	case MODE_V8DF:
	case MODE_V16SF:
	  gcc_assert (TARGET_AVX512F);
	  return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";

	case MODE_OI:
	case MODE_V4DF:
	case MODE_V8SF:
	  gcc_assert (TARGET_AVX2);
	  /* FALLTHRU */
	case MODE_TI:
	case MODE_V2DF:
	case MODE_V4SF:
	  gcc_assert (TARGET_SSE2);
	  return (TARGET_AVX
		  ? "vpcmpeqd\t%0, %0, %0"
		  : "pcmpeqd\t%0, %0");

	default:
	  gcc_unreachable ();
	}
   }

  gcc_unreachable ();
}

/* Returns true if INSN can be transformed from a memory load
   to a supported FP constant load.  */

bool
ix86_standard_x87sse_constant_load_p (const rtx_insn *insn, rtx dst)
{
  rtx src = find_constant_src (insn);

  gcc_assert (REG_P (dst));

  if (src == NULL
      || (SSE_REGNO_P (REGNO (dst))
	  && standard_sse_constant_p (src, GET_MODE (dst)) != 1)
      || (STACK_REGNO_P (REGNO (dst))
	   && standard_80387_constant_p (src) < 1))
    return false;

  return true;
}

/* Returns true if OP contains a symbol reference */

bool
symbolic_reference_mentioned_p (rtx op)
{
  const char *fmt;
  int i;

  if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF)
    return true;

  fmt = GET_RTX_FORMAT (GET_CODE (op));
  for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--)
    {
      if (fmt[i] == 'E')
	{
	  int j;

	  for (j = XVECLEN (op, i) - 1; j >= 0; j--)
	    if (symbolic_reference_mentioned_p (XVECEXP (op, i, j)))
	      return true;
	}

      else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i)))
	return true;
    }

  return false;
}

/* Return true if it is appropriate to emit `ret' instructions in the
   body of a function.  Do this only if the epilogue is simple, needing a
   couple of insns.  Prior to reloading, we can't tell how many registers
   must be saved, so return false then.  Return false if there is no frame
   marker to de-allocate.  */

bool
ix86_can_use_return_insn_p (void)
{
  struct ix86_frame frame;

  /* Don't use `ret' instruction in interrupt handler.  */
  if (! reload_completed
      || frame_pointer_needed
      || cfun->machine->func_type != TYPE_NORMAL)
    return 0;

  /* Don't allow more than 32k pop, since that's all we can do
     with one instruction.  */
  if (crtl->args.pops_args && crtl->args.size >= 32768)
    return 0;

  ix86_compute_frame_layout (&frame);
  return (frame.stack_pointer_offset == UNITS_PER_WORD
	  && (frame.nregs + frame.nsseregs) == 0);
}

/* Value should be nonzero if functions must have frame pointers.
   Zero means the frame pointer need not be set up (and parms may
   be accessed via the stack pointer) in functions that seem suitable.  */

static bool
ix86_frame_pointer_required (void)
{
  /* If we accessed previous frames, then the generated code expects
     to be able to access the saved ebp value in our frame.  */
  if (cfun->machine->accesses_prev_frame)
    return true;

  /* Several x86 os'es need a frame pointer for other reasons,
     usually pertaining to setjmp.  */
  if (SUBTARGET_FRAME_POINTER_REQUIRED)
    return true;

  /* For older 32-bit runtimes setjmp requires valid frame-pointer.  */
  if (TARGET_32BIT_MS_ABI && cfun->calls_setjmp)
    return true;

  /* Win64 SEH, very large frames need a frame-pointer as maximum stack
     allocation is 4GB.  */
  if (TARGET_64BIT_MS_ABI && get_frame_size () > SEH_MAX_FRAME_SIZE)
    return true;

  /* SSE saves require frame-pointer when stack is misaligned.  */
  if (TARGET_64BIT_MS_ABI && ix86_incoming_stack_boundary < 128)
    return true;
  
  /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
     turns off the frame pointer by default.  Turn it back on now if
     we've not got a leaf function.  */
  if (TARGET_OMIT_LEAF_FRAME_POINTER
      && (!crtl->is_leaf
	  || ix86_current_function_calls_tls_descriptor))
    return true;

  if (crtl->profile && !flag_fentry)
    return true;

  return false;
}

/* Record that the current function accesses previous call frames.  */

void
ix86_setup_frame_addresses (void)
{
  cfun->machine->accesses_prev_frame = 1;
}

#ifndef USE_HIDDEN_LINKONCE
# if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
#  define USE_HIDDEN_LINKONCE 1
# else
#  define USE_HIDDEN_LINKONCE 0
# endif
#endif

static int pic_labels_used;

/* Fills in the label name that should be used for a pc thunk for
   the given register.  */

static void
get_pc_thunk_name (char name[32], unsigned int regno)
{
  gcc_assert (!TARGET_64BIT);

  if (USE_HIDDEN_LINKONCE)
    sprintf (name, "__x86.get_pc_thunk.%s", reg_names[regno]);
  else
    ASM_GENERATE_INTERNAL_LABEL (name, "LPR", regno);
}


/* This function generates code for -fpic that loads %ebx with
   the return address of the caller and then returns.  */

static void
ix86_code_end (void)
{
  rtx xops[2];
  int regno;

  for (regno = AX_REG; regno <= SP_REG; regno++)
    {
      char name[32];
      tree decl;

      if (!(pic_labels_used & (1 << regno)))
	continue;

      get_pc_thunk_name (name, regno);

      decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
			 get_identifier (name),
			 build_function_type_list (void_type_node, NULL_TREE));
      DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
				       NULL_TREE, void_type_node);
      TREE_PUBLIC (decl) = 1;
      TREE_STATIC (decl) = 1;
      DECL_IGNORED_P (decl) = 1;

#if TARGET_MACHO
      if (TARGET_MACHO)
	{
	  switch_to_section (darwin_sections[picbase_thunk_section]);
	  fputs ("\t.weak_definition\t", asm_out_file);
	  assemble_name (asm_out_file, name);
	  fputs ("\n\t.private_extern\t", asm_out_file);
	  assemble_name (asm_out_file, name);
	  putc ('\n', asm_out_file);
	  ASM_OUTPUT_LABEL (asm_out_file, name);
	  DECL_WEAK (decl) = 1;
	}
      else
#endif
      if (USE_HIDDEN_LINKONCE)
	{
	  cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));

	  targetm.asm_out.unique_section (decl, 0);
	  switch_to_section (get_named_section (decl, NULL, 0));

	  targetm.asm_out.globalize_label (asm_out_file, name);
	  fputs ("\t.hidden\t", asm_out_file);
	  assemble_name (asm_out_file, name);
	  putc ('\n', asm_out_file);
	  ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
	}
      else
	{
	  switch_to_section (text_section);
	  ASM_OUTPUT_LABEL (asm_out_file, name);
	}

      DECL_INITIAL (decl) = make_node (BLOCK);
      current_function_decl = decl;
      allocate_struct_function (decl, false);
      init_function_start (decl);
      /* We're about to hide the function body from callees of final_* by
	 emitting it directly; tell them we're a thunk, if they care.  */
      cfun->is_thunk = true;
      first_function_block_is_cold = false;
      /* Make sure unwind info is emitted for the thunk if needed.  */
      final_start_function (emit_barrier (), asm_out_file, 1);

      /* Pad stack IP move with 4 instructions (two NOPs count
	 as one instruction).  */
      if (TARGET_PAD_SHORT_FUNCTION)
	{
	  int i = 8;

	  while (i--)
	    fputs ("\tnop\n", asm_out_file);
	}

      xops[0] = gen_rtx_REG (Pmode, regno);
      xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
      output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
      output_asm_insn ("%!ret", NULL);
      final_end_function ();
      init_insn_lengths ();
      free_after_compilation (cfun);
      set_cfun (NULL);
      current_function_decl = NULL;
    }

  if (flag_split_stack)
    file_end_indicate_split_stack ();
}

/* Emit code for the SET_GOT patterns.  */

const char *
output_set_got (rtx dest, rtx label)
{
  rtx xops[3];

  xops[0] = dest;

  if (TARGET_VXWORKS_RTP && flag_pic)
    {
      /* Load (*VXWORKS_GOTT_BASE) into the PIC register.  */
      xops[2] = gen_rtx_MEM (Pmode,
			     gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_BASE));
      output_asm_insn ("mov{l}\t{%2, %0|%0, %2}", xops);

      /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
	 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
	 an unadorned address.  */
      xops[2] = gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_INDEX);
      SYMBOL_REF_FLAGS (xops[2]) |= SYMBOL_FLAG_LOCAL;
      output_asm_insn ("mov{l}\t{%P2(%0), %0|%0, DWORD PTR %P2[%0]}", xops);
      return "";
    }

  xops[1] = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);

  if (flag_pic)
    {
      char name[32];
      get_pc_thunk_name (name, REGNO (dest));
      pic_labels_used |= 1 << REGNO (dest);

      xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
      xops[2] = gen_rtx_MEM (QImode, xops[2]);
      output_asm_insn ("%!call\t%X2", xops);

#if TARGET_MACHO
      /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
         This is what will be referenced by the Mach-O PIC subsystem.  */
      if (machopic_should_output_picbase_label () || !label)
	ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);

      /* When we are restoring the pic base at the site of a nonlocal label,
         and we decided to emit the pic base above, we will still output a
         local label used for calculating the correction offset (even though
         the offset will be 0 in that case).  */
      if (label)
        targetm.asm_out.internal_label (asm_out_file, "L",
					   CODE_LABEL_NUMBER (label));
#endif
    }
  else
    {
      if (TARGET_MACHO)
	/* We don't need a pic base, we're not producing pic.  */
	gcc_unreachable ();

      xops[2] = gen_rtx_LABEL_REF (Pmode, label ? label : gen_label_rtx ());
      output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops);
      targetm.asm_out.internal_label (asm_out_file, "L",
				      CODE_LABEL_NUMBER (XEXP (xops[2], 0)));
    }

  if (!TARGET_MACHO)
    output_asm_insn ("add%z0\t{%1, %0|%0, %1}", xops);

  return "";
}

/* Generate an "push" pattern for input ARG.  */

static rtx
gen_push (rtx arg)
{
  struct machine_function *m = cfun->machine;

  if (m->fs.cfa_reg == stack_pointer_rtx)
    m->fs.cfa_offset += UNITS_PER_WORD;
  m->fs.sp_offset += UNITS_PER_WORD;

  if (REG_P (arg) && GET_MODE (arg) != word_mode)
    arg = gen_rtx_REG (word_mode, REGNO (arg));

  return gen_rtx_SET (gen_rtx_MEM (word_mode,
				   gen_rtx_PRE_DEC (Pmode,
						    stack_pointer_rtx)),
		      arg);
}

/* Generate an "pop" pattern for input ARG.  */

static rtx
gen_pop (rtx arg)
{
  if (REG_P (arg) && GET_MODE (arg) != word_mode)
    arg = gen_rtx_REG (word_mode, REGNO (arg));

  return gen_rtx_SET (arg,
		      gen_rtx_MEM (word_mode,
				   gen_rtx_POST_INC (Pmode,
						     stack_pointer_rtx)));
}

/* Return >= 0 if there is an unused call-clobbered register available
   for the entire function.  */

static unsigned int
ix86_select_alt_pic_regnum (void)
{
  if (ix86_use_pseudo_pic_reg ())
    return INVALID_REGNUM;

  if (crtl->is_leaf
      && !crtl->profile
      && !ix86_current_function_calls_tls_descriptor)
    {
      int i, drap;
      /* Can't use the same register for both PIC and DRAP.  */
      if (crtl->drap_reg)
	drap = REGNO (crtl->drap_reg);
      else
	drap = -1;
      for (i = 2; i >= 0; --i)
        if (i != drap && !df_regs_ever_live_p (i))
	  return i;
    }

  return INVALID_REGNUM;
}

/* Return true if REGNO is used by the epilogue.  */

bool
ix86_epilogue_uses (int regno)
{
  /* If there are no caller-saved registers, we preserve all registers,
     except for MMX and x87 registers which aren't supported when saving
     and restoring registers.  Don't explicitly save SP register since
     it is always preserved.  */
  return (epilogue_completed
	  && cfun->machine->no_caller_saved_registers
	  && !fixed_regs[regno]
	  && !STACK_REGNO_P (regno)
	  && !MMX_REGNO_P (regno));
}

/* Return nonzero if register REGNO can be used as a scratch register
   in peephole2.  */

static bool
ix86_hard_regno_scratch_ok (unsigned int regno)
{
  /* If there are no caller-saved registers, we can't use any register
     as a scratch register after epilogue and use REGNO as scratch
     register only if it has been used before to avoid saving and
     restoring it.  */
  return (!cfun->machine->no_caller_saved_registers
	  || (!epilogue_completed
	      && df_regs_ever_live_p (regno)));
}

/* Return true if register class CL should be an additional allocno
   class.  */

static bool
ix86_additional_allocno_class_p (reg_class_t cl)
{
  return cl == MOD4_SSE_REGS;
}

/* Return TRUE if we need to save REGNO.  */

static bool
ix86_save_reg (unsigned int regno, bool maybe_eh_return)
{
  /* If there are no caller-saved registers, we preserve all registers,
     except for MMX and x87 registers which aren't supported when saving
     and restoring registers.  Don't explicitly save SP register since
     it is always preserved.  */
  if (cfun->machine->no_caller_saved_registers)
    {
      /* Don't preserve registers used for function return value.  */
      rtx reg = crtl->return_rtx;
      if (reg)
	{
	  unsigned int i = REGNO (reg);
	  unsigned int nregs = hard_regno_nregs[i][GET_MODE (reg)];
	  while (nregs-- > 0)
	    if ((i + nregs) == regno)
	      return false;

	  reg = crtl->return_bnd;
	  if (reg)
	    {
	      i = REGNO (reg);
	      nregs = hard_regno_nregs[i][GET_MODE (reg)];
	      while (nregs-- > 0)
		if ((i + nregs) == regno)
		  return false;
	    }
	}

      return (df_regs_ever_live_p (regno)
	      && !fixed_regs[regno]
	      && !STACK_REGNO_P (regno)
	      && !MMX_REGNO_P (regno)
	      && (regno != HARD_FRAME_POINTER_REGNUM
		  || !frame_pointer_needed));
    }

  if (regno == REAL_PIC_OFFSET_TABLE_REGNUM
      && pic_offset_table_rtx)
    {
      if (ix86_use_pseudo_pic_reg ())
	{
	  /* REAL_PIC_OFFSET_TABLE_REGNUM used by call to
	  _mcount in prologue.  */
	  if (!TARGET_64BIT && flag_pic && crtl->profile)
	    return true;
	}
      else if (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
	       || crtl->profile
	       || crtl->calls_eh_return
	       || crtl->uses_const_pool
	       || cfun->has_nonlocal_label)
        return ix86_select_alt_pic_regnum () == INVALID_REGNUM;
    }

  if (crtl->calls_eh_return && maybe_eh_return)
    {
      unsigned i;
      for (i = 0; ; i++)
	{
	  unsigned test = EH_RETURN_DATA_REGNO (i);
	  if (test == INVALID_REGNUM)
	    break;
	  if (test == regno)
	    return true;
	}
    }

  if (crtl->drap_reg
      && regno == REGNO (crtl->drap_reg)
      && !cfun->machine->no_drap_save_restore)
    return true;

  return (df_regs_ever_live_p (regno)
	  && !call_used_regs[regno]
	  && !fixed_regs[regno]
	  && (regno != HARD_FRAME_POINTER_REGNUM || !frame_pointer_needed));
}

/* Return number of saved general prupose registers.  */

static int
ix86_nsaved_regs (void)
{
  int nregs = 0;
  int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true))
      nregs ++;
  return nregs;
}

/* Return number of saved SSE registers.  */

static int
ix86_nsaved_sseregs (void)
{
  int nregs = 0;
  int regno;

  if (!TARGET_64BIT_MS_ABI)
    return 0;
  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
      nregs ++;
  return nregs;
}

/* Given FROM and TO register numbers, say whether this elimination is
   allowed.  If stack alignment is needed, we can only replace argument
   pointer with hard frame pointer, or replace frame pointer with stack
   pointer.  Otherwise, frame pointer elimination is automatically
   handled and all other eliminations are valid.  */

static bool
ix86_can_eliminate (const int from, const int to)
{
  if (stack_realign_fp)
    return ((from == ARG_POINTER_REGNUM
	     && to == HARD_FRAME_POINTER_REGNUM)
	    || (from == FRAME_POINTER_REGNUM
		&& to == STACK_POINTER_REGNUM));
  else
    return to == STACK_POINTER_REGNUM ? !frame_pointer_needed : true;
}

/* Return the offset between two registers, one to be eliminated, and the other
   its replacement, at the start of a routine.  */

HOST_WIDE_INT
ix86_initial_elimination_offset (int from, int to)
{
  struct ix86_frame frame;
  ix86_compute_frame_layout (&frame);

  if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM)
    return frame.hard_frame_pointer_offset;
  else if (from == FRAME_POINTER_REGNUM
	   && to == HARD_FRAME_POINTER_REGNUM)
    return frame.hard_frame_pointer_offset - frame.frame_pointer_offset;
  else
    {
      gcc_assert (to == STACK_POINTER_REGNUM);

      if (from == ARG_POINTER_REGNUM)
	return frame.stack_pointer_offset;

      gcc_assert (from == FRAME_POINTER_REGNUM);
      return frame.stack_pointer_offset - frame.frame_pointer_offset;
    }
}

/* In a dynamically-aligned function, we can't know the offset from
   stack pointer to frame pointer, so we must ensure that setjmp
   eliminates fp against the hard fp (%ebp) rather than trying to
   index from %esp up to the top of the frame across a gap that is
   of unknown (at compile-time) size.  */
static rtx
ix86_builtin_setjmp_frame_value (void)
{
  return stack_realign_fp ? hard_frame_pointer_rtx : virtual_stack_vars_rtx;
}

/* When using -fsplit-stack, the allocation routines set a field in
   the TCB to the bottom of the stack plus this much space, measured
   in bytes.  */

#define SPLIT_STACK_AVAILABLE 256

/* Fill structure ix86_frame about frame of currently computed function.  */

static void
ix86_compute_frame_layout (struct ix86_frame *frame)
{
  unsigned HOST_WIDE_INT stack_alignment_needed;
  HOST_WIDE_INT offset;
  unsigned HOST_WIDE_INT preferred_alignment;
  HOST_WIDE_INT size = get_frame_size ();
  HOST_WIDE_INT to_allocate;

  frame->nregs = ix86_nsaved_regs ();
  frame->nsseregs = ix86_nsaved_sseregs ();

  /* 64-bit MS ABI seem to require stack alignment to be always 16,
     except for function prologues, leaf functions and when the defult
     incoming stack boundary is overriden at command line or via
     force_align_arg_pointer attribute.  */
  if ((TARGET_64BIT_MS_ABI && crtl->preferred_stack_boundary < 128)
      && (!crtl->is_leaf || cfun->calls_alloca != 0
	  || ix86_current_function_calls_tls_descriptor
	  || ix86_incoming_stack_boundary < 128))
    {
      crtl->preferred_stack_boundary = 128;
      crtl->stack_alignment_needed = 128;
    }

  stack_alignment_needed = crtl->stack_alignment_needed / BITS_PER_UNIT;
  preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT;

  gcc_assert (!size || stack_alignment_needed);
  gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
  gcc_assert (preferred_alignment <= stack_alignment_needed);

  /* For SEH we have to limit the amount of code movement into the prologue.
     At present we do this via a BLOCKAGE, at which point there's very little
     scheduling that can be done, which means that there's very little point
     in doing anything except PUSHs.  */
  if (TARGET_SEH)
    cfun->machine->use_fast_prologue_epilogue = false;

  /* During reload iteration the amount of registers saved can change.
     Recompute the value as needed.  Do not recompute when amount of registers
     didn't change as reload does multiple calls to the function and does not
     expect the decision to change within single iteration.  */
  else if (!optimize_bb_for_size_p (ENTRY_BLOCK_PTR_FOR_FN (cfun))
           && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs)
    {
      int count = frame->nregs;
      struct cgraph_node *node = cgraph_node::get (current_function_decl);

      cfun->machine->use_fast_prologue_epilogue_nregs = count;

      /* The fast prologue uses move instead of push to save registers.  This
         is significantly longer, but also executes faster as modern hardware
         can execute the moves in parallel, but can't do that for push/pop.

	 Be careful about choosing what prologue to emit:  When function takes
	 many instructions to execute we may use slow version as well as in
	 case function is known to be outside hot spot (this is known with
	 feedback only).  Weight the size of function by number of registers
	 to save as it is cheap to use one or two push instructions but very
	 slow to use many of them.  */
      if (count)
	count = (count - 1) * FAST_PROLOGUE_INSN_COUNT;
      if (node->frequency < NODE_FREQUENCY_NORMAL
	  || (flag_branch_probabilities
	      && node->frequency < NODE_FREQUENCY_HOT))
        cfun->machine->use_fast_prologue_epilogue = false;
      else
        cfun->machine->use_fast_prologue_epilogue
	   = !expensive_function_p (count);
    }

  frame->save_regs_using_mov
    = (TARGET_PROLOGUE_USING_MOVE && cfun->machine->use_fast_prologue_epilogue
       /* If static stack checking is enabled and done with probes,
	  the registers need to be saved before allocating the frame.  */
       && flag_stack_check != STATIC_BUILTIN_STACK_CHECK);

  /* Skip return address.  */
  offset = UNITS_PER_WORD;

  /* Skip pushed static chain.  */
  if (ix86_static_chain_on_stack)
    offset += UNITS_PER_WORD;

  /* Skip saved base pointer.  */
  if (frame_pointer_needed)
    offset += UNITS_PER_WORD;
  frame->hfp_save_offset = offset;

  /* The traditional frame pointer location is at the top of the frame.  */
  frame->hard_frame_pointer_offset = offset;

  /* Register save area */
  offset += frame->nregs * UNITS_PER_WORD;
  frame->reg_save_offset = offset;

  /* On SEH target, registers are pushed just before the frame pointer
     location.  */
  if (TARGET_SEH)
    frame->hard_frame_pointer_offset = offset;

  /* Align and set SSE register save area.  */
  if (frame->nsseregs)
    {
      /* The only ABI that has saved SSE registers (Win64) also has a
	 16-byte aligned default stack, and thus we don't need to be
	 within the re-aligned local stack frame to save them.  In case
	 incoming stack boundary is aligned to less than 16 bytes,
	 unaligned move of SSE register will be emitted, so there is
	 no point to round up the SSE register save area outside the
	 re-aligned local stack frame to 16 bytes.  */
      if (ix86_incoming_stack_boundary >= 128)
	offset = ROUND_UP (offset, 16);
      offset += frame->nsseregs * 16;
    }
  frame->sse_reg_save_offset = offset;

  /* The re-aligned stack starts here.  Values before this point are not
     directly comparable with values below this point.  In order to make
     sure that no value happens to be the same before and after, force
     the alignment computation below to add a non-zero value.  */
  if (stack_realign_fp)
    offset = ROUND_UP (offset, stack_alignment_needed);

  /* Va-arg area */
  frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size;
  offset += frame->va_arg_size;

  /* Align start of frame for local function.  */
  if (stack_realign_fp
      || offset != frame->sse_reg_save_offset
      || size != 0
      || !crtl->is_leaf
      || cfun->calls_alloca
      || ix86_current_function_calls_tls_descriptor)
    offset = ROUND_UP (offset, stack_alignment_needed);

  /* Frame pointer points here.  */
  frame->frame_pointer_offset = offset;

  offset += size;

  /* Add outgoing arguments area.  Can be skipped if we eliminated
     all the function calls as dead code.
     Skipping is however impossible when function calls alloca.  Alloca
     expander assumes that last crtl->outgoing_args_size
     of stack frame are unused.  */
  if (ACCUMULATE_OUTGOING_ARGS
      && (!crtl->is_leaf || cfun->calls_alloca
	  || ix86_current_function_calls_tls_descriptor))
    {
      offset += crtl->outgoing_args_size;
      frame->outgoing_arguments_size = crtl->outgoing_args_size;
    }
  else
    frame->outgoing_arguments_size = 0;

  /* Align stack boundary.  Only needed if we're calling another function
     or using alloca.  */
  if (!crtl->is_leaf || cfun->calls_alloca
      || ix86_current_function_calls_tls_descriptor)
    offset = ROUND_UP (offset, preferred_alignment);

  /* We've reached end of stack frame.  */
  frame->stack_pointer_offset = offset;

  /* Size prologue needs to allocate.  */
  to_allocate = offset - frame->sse_reg_save_offset;

  if ((!to_allocate && frame->nregs <= 1)
      || (TARGET_64BIT && to_allocate >= HOST_WIDE_INT_C (0x80000000)))
    frame->save_regs_using_mov = false;

  if (ix86_using_red_zone ()
      && crtl->sp_is_unchanging
      && crtl->is_leaf
      && !ix86_pc_thunk_call_expanded
      && !ix86_current_function_calls_tls_descriptor)
    {
      frame->red_zone_size = to_allocate;
      if (frame->save_regs_using_mov)
	frame->red_zone_size += frame->nregs * UNITS_PER_WORD;
      if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE)
	frame->red_zone_size = RED_ZONE_SIZE - RED_ZONE_RESERVE;
    }
  else
    frame->red_zone_size = 0;
  frame->stack_pointer_offset -= frame->red_zone_size;

  /* The SEH frame pointer location is near the bottom of the frame.
     This is enforced by the fact that the difference between the
     stack pointer and the frame pointer is limited to 240 bytes in
     the unwind data structure.  */
  if (TARGET_SEH)
    {
      HOST_WIDE_INT diff;

      /* If we can leave the frame pointer where it is, do so.  Also, returns
	 the establisher frame for __builtin_frame_address (0).  */
      diff = frame->stack_pointer_offset - frame->hard_frame_pointer_offset;
      if (diff <= SEH_MAX_FRAME_SIZE
	  && (diff > 240 || (diff & 15) != 0)
	  && !crtl->accesses_prior_frames)
	{
	  /* Ideally we'd determine what portion of the local stack frame
	     (within the constraint of the lowest 240) is most heavily used.
	     But without that complication, simply bias the frame pointer
	     by 128 bytes so as to maximize the amount of the local stack
	     frame that is addressable with 8-bit offsets.  */
	  frame->hard_frame_pointer_offset = frame->stack_pointer_offset - 128;
	}
    }
}

/* This is semi-inlined memory_address_length, but simplified
   since we know that we're always dealing with reg+offset, and
   to avoid having to create and discard all that rtl.  */

static inline int
choose_baseaddr_len (unsigned int regno, HOST_WIDE_INT offset)
{
  int len = 4;

  if (offset == 0)
    {
      /* EBP and R13 cannot be encoded without an offset.  */
      len = (regno == BP_REG || regno == R13_REG);
    }
  else if (IN_RANGE (offset, -128, 127))
    len = 1;

  /* ESP and R12 must be encoded with a SIB byte.  */
  if (regno == SP_REG || regno == R12_REG)
    len++;

  return len;
}

/* Return an RTX that points to CFA_OFFSET within the stack frame.
   The valid base registers are taken from CFUN->MACHINE->FS.  */

static rtx
choose_baseaddr (HOST_WIDE_INT cfa_offset)
{
  const struct machine_function *m = cfun->machine;
  rtx base_reg = NULL;
  HOST_WIDE_INT base_offset = 0;

  if (m->use_fast_prologue_epilogue)
    {
      /* Choose the base register most likely to allow the most scheduling
         opportunities.  Generally FP is valid throughout the function,
         while DRAP must be reloaded within the epilogue.  But choose either
         over the SP due to increased encoding size.  */

      if (m->fs.fp_valid)
	{
	  base_reg = hard_frame_pointer_rtx;
	  base_offset = m->fs.fp_offset - cfa_offset;
	}
      else if (m->fs.drap_valid)
	{
	  base_reg = crtl->drap_reg;
	  base_offset = 0 - cfa_offset;
	}
      else if (m->fs.sp_valid)
	{
	  base_reg = stack_pointer_rtx;
	  base_offset = m->fs.sp_offset - cfa_offset;
	}
    }
  else
    {
      HOST_WIDE_INT toffset;
      int len = 16, tlen;

      /* Choose the base register with the smallest address encoding.
         With a tie, choose FP > DRAP > SP.  */
      if (m->fs.sp_valid)
	{
	  base_reg = stack_pointer_rtx;
	  base_offset = m->fs.sp_offset - cfa_offset;
          len = choose_baseaddr_len (STACK_POINTER_REGNUM, base_offset);
	}
      if (m->fs.drap_valid)
	{
	  toffset = 0 - cfa_offset;
	  tlen = choose_baseaddr_len (REGNO (crtl->drap_reg), toffset);
	  if (tlen <= len)
	    {
	      base_reg = crtl->drap_reg;
	      base_offset = toffset;
	      len = tlen;
	    }
	}
      if (m->fs.fp_valid)
	{
	  toffset = m->fs.fp_offset - cfa_offset;
	  tlen = choose_baseaddr_len (HARD_FRAME_POINTER_REGNUM, toffset);
	  if (tlen <= len)
	    {
	      base_reg = hard_frame_pointer_rtx;
	      base_offset = toffset;
	      len = tlen;
	    }
	}
    }
  gcc_assert (base_reg != NULL);

  return plus_constant (Pmode, base_reg, base_offset);
}

/* Emit code to save registers in the prologue.  */

static void
ix86_emit_save_regs (void)
{
  unsigned int regno;
  rtx_insn *insn;

  for (regno = FIRST_PSEUDO_REGISTER - 1; regno-- > 0; )
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true))
      {
	insn = emit_insn (gen_push (gen_rtx_REG (word_mode, regno)));
	RTX_FRAME_RELATED_P (insn) = 1;
      }
}

/* Emit a single register save at CFA - CFA_OFFSET.  */

static void
ix86_emit_save_reg_using_mov (machine_mode mode, unsigned int regno,
			      HOST_WIDE_INT cfa_offset)
{
  struct machine_function *m = cfun->machine;
  rtx reg = gen_rtx_REG (mode, regno);
  rtx mem, addr, base, insn;
  unsigned int align;

  addr = choose_baseaddr (cfa_offset);
  mem = gen_frame_mem (mode, addr);

  /* The location is aligned up to INCOMING_STACK_BOUNDARY.  */
  align = MIN (GET_MODE_ALIGNMENT (mode), INCOMING_STACK_BOUNDARY);
  set_mem_align (mem, align);

  insn = emit_insn (gen_rtx_SET (mem, reg));
  RTX_FRAME_RELATED_P (insn) = 1;

  base = addr;
  if (GET_CODE (base) == PLUS)
    base = XEXP (base, 0);
  gcc_checking_assert (REG_P (base));

  /* When saving registers into a re-aligned local stack frame, avoid
     any tricky guessing by dwarf2out.  */
  if (m->fs.realigned)
    {
      gcc_checking_assert (stack_realign_drap);

      if (regno == REGNO (crtl->drap_reg))
	{
	  /* A bit of a hack.  We force the DRAP register to be saved in
	     the re-aligned stack frame, which provides us with a copy
	     of the CFA that will last past the prologue.  Install it.  */
	  gcc_checking_assert (cfun->machine->fs.fp_valid);
	  addr = plus_constant (Pmode, hard_frame_pointer_rtx,
				cfun->machine->fs.fp_offset - cfa_offset);
	  mem = gen_rtx_MEM (mode, addr);
	  add_reg_note (insn, REG_CFA_DEF_CFA, mem);
	}
      else
	{
	  /* The frame pointer is a stable reference within the
	     aligned frame.  Use it.  */
	  gcc_checking_assert (cfun->machine->fs.fp_valid);
	  addr = plus_constant (Pmode, hard_frame_pointer_rtx,
				cfun->machine->fs.fp_offset - cfa_offset);
	  mem = gen_rtx_MEM (mode, addr);
	  add_reg_note (insn, REG_CFA_EXPRESSION, gen_rtx_SET (mem, reg));
	}
    }

  /* The memory may not be relative to the current CFA register,
     which means that we may need to generate a new pattern for
     use by the unwind info.  */
  else if (base != m->fs.cfa_reg)
    {
      addr = plus_constant (Pmode, m->fs.cfa_reg,
			    m->fs.cfa_offset - cfa_offset);
      mem = gen_rtx_MEM (mode, addr);
      add_reg_note (insn, REG_CFA_OFFSET, gen_rtx_SET (mem, reg));
    }
}

/* Emit code to save registers using MOV insns.
   First register is stored at CFA - CFA_OFFSET.  */
static void
ix86_emit_save_regs_using_mov (HOST_WIDE_INT cfa_offset)
{
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true))
      {
        ix86_emit_save_reg_using_mov (word_mode, regno, cfa_offset);
	cfa_offset -= UNITS_PER_WORD;
      }
}

/* Emit code to save SSE registers using MOV insns.
   First register is stored at CFA - CFA_OFFSET.  */
static void
ix86_emit_save_sse_regs_using_mov (HOST_WIDE_INT cfa_offset)
{
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
      {
	ix86_emit_save_reg_using_mov (V4SFmode, regno, cfa_offset);
	cfa_offset -= GET_MODE_SIZE (V4SFmode);
      }
}

static GTY(()) rtx queued_cfa_restores;

/* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
   manipulation insn.  The value is on the stack at CFA - CFA_OFFSET.
   Don't add the note if the previously saved value will be left untouched
   within stack red-zone till return, as unwinders can find the same value
   in the register and on the stack.  */

static void
ix86_add_cfa_restore_note (rtx_insn *insn, rtx reg, HOST_WIDE_INT cfa_offset)
{
  if (!crtl->shrink_wrapped
      && cfa_offset <= cfun->machine->fs.red_zone_offset)
    return;

  if (insn)
    {
      add_reg_note (insn, REG_CFA_RESTORE, reg);
      RTX_FRAME_RELATED_P (insn) = 1;
    }
  else
    queued_cfa_restores
      = alloc_reg_note (REG_CFA_RESTORE, reg, queued_cfa_restores);
}

/* Add queued REG_CFA_RESTORE notes if any to INSN.  */

static void
ix86_add_queued_cfa_restore_notes (rtx insn)
{
  rtx last;
  if (!queued_cfa_restores)
    return;
  for (last = queued_cfa_restores; XEXP (last, 1); last = XEXP (last, 1))
    ;
  XEXP (last, 1) = REG_NOTES (insn);
  REG_NOTES (insn) = queued_cfa_restores;
  queued_cfa_restores = NULL_RTX;
  RTX_FRAME_RELATED_P (insn) = 1;
}

/* Expand prologue or epilogue stack adjustment.
   The pattern exist to put a dependency on all ebp-based memory accesses.
   STYLE should be negative if instructions should be marked as frame related,
   zero if %r11 register is live and cannot be freely used and positive
   otherwise.  */

static void
pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset,
			   int style, bool set_cfa)
{
  struct machine_function *m = cfun->machine;
  rtx insn;
  bool add_frame_related_expr = false;

  if (Pmode == SImode)
    insn = gen_pro_epilogue_adjust_stack_si_add (dest, src, offset);
  else if (x86_64_immediate_operand (offset, DImode))
    insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, offset);
  else
    {
      rtx tmp;
      /* r11 is used by indirect sibcall return as well, set before the
	 epilogue and used after the epilogue.  */
      if (style)
        tmp = gen_rtx_REG (DImode, R11_REG);
      else
	{
	  gcc_assert (src != hard_frame_pointer_rtx
		      && dest != hard_frame_pointer_rtx);
	  tmp = hard_frame_pointer_rtx;
	}
      insn = emit_insn (gen_rtx_SET (tmp, offset));
      if (style < 0)
	add_frame_related_expr = true;

      insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, tmp);
    }

  insn = emit_insn (insn);
  if (style >= 0)
    ix86_add_queued_cfa_restore_notes (insn);

  if (set_cfa)
    {
      rtx r;

      gcc_assert (m->fs.cfa_reg == src);
      m->fs.cfa_offset += INTVAL (offset);
      m->fs.cfa_reg = dest;

      r = gen_rtx_PLUS (Pmode, src, offset);
      r = gen_rtx_SET (dest, r);
      add_reg_note (insn, REG_CFA_ADJUST_CFA, r);
      RTX_FRAME_RELATED_P (insn) = 1;
    }
  else if (style < 0)
    {
      RTX_FRAME_RELATED_P (insn) = 1;
      if (add_frame_related_expr)
	{
	  rtx r = gen_rtx_PLUS (Pmode, src, offset);
	  r = gen_rtx_SET (dest, r);
	  add_reg_note (insn, REG_FRAME_RELATED_EXPR, r);
	}
    }

  if (dest == stack_pointer_rtx)
    {
      HOST_WIDE_INT ooffset = m->fs.sp_offset;
      bool valid = m->fs.sp_valid;

      if (src == hard_frame_pointer_rtx)
	{
	  valid = m->fs.fp_valid;
	  ooffset = m->fs.fp_offset;
	}
      else if (src == crtl->drap_reg)
	{
	  valid = m->fs.drap_valid;
	  ooffset = 0;
	}
      else
	{
	  /* Else there are two possibilities: SP itself, which we set
	     up as the default above.  Or EH_RETURN_STACKADJ_RTX, which is
	     taken care of this by hand along the eh_return path.  */
	  gcc_checking_assert (src == stack_pointer_rtx
			       || offset == const0_rtx);
	}

      m->fs.sp_offset = ooffset - INTVAL (offset);
      m->fs.sp_valid = valid;
    }
}

/* Find an available register to be used as dynamic realign argument
   pointer regsiter.  Such a register will be written in prologue and
   used in begin of body, so it must not be
	1. parameter passing register.
	2. GOT pointer.
   We reuse static-chain register if it is available.  Otherwise, we
   use DI for i386 and R13 for x86-64.  We chose R13 since it has
   shorter encoding.

   Return: the regno of chosen register.  */

static unsigned int
find_drap_reg (void)
{
  tree decl = cfun->decl;

  /* Always use callee-saved register if there are no caller-saved
     registers.  */
  if (TARGET_64BIT)
    {
      /* Use R13 for nested function or function need static chain.
	 Since function with tail call may use any caller-saved
	 registers in epilogue, DRAP must not use caller-saved
	 register in such case.  */
      if (DECL_STATIC_CHAIN (decl)
	  || cfun->machine->no_caller_saved_registers
	  || crtl->tail_call_emit)
	return R13_REG;

      return R10_REG;
    }
  else
    {
      /* Use DI for nested function or function need static chain.
	 Since function with tail call may use any caller-saved
	 registers in epilogue, DRAP must not use caller-saved
	 register in such case.  */
      if (DECL_STATIC_CHAIN (decl)
	  || cfun->machine->no_caller_saved_registers
	  || crtl->tail_call_emit)
	return DI_REG;

      /* Reuse static chain register if it isn't used for parameter
         passing.  */
      if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2)
	{
	  unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (decl));
	  if ((ccvt & (IX86_CALLCVT_FASTCALL | IX86_CALLCVT_THISCALL)) == 0)
	    return CX_REG;
	}
      return DI_REG;
    }
}

/* Handle a "force_align_arg_pointer" attribute.  */

static tree
ix86_handle_force_align_arg_pointer_attribute (tree *node, tree name,
					       tree, int, bool *no_add_attrs)
{
  if (TREE_CODE (*node) != FUNCTION_TYPE
      && TREE_CODE (*node) != METHOD_TYPE
      && TREE_CODE (*node) != FIELD_DECL
      && TREE_CODE (*node) != TYPE_DECL)
    {
      warning (OPT_Wattributes, "%qE attribute only applies to functions",
	       name);
      *no_add_attrs = true;
    }

  return NULL_TREE;
}

/* Return minimum incoming stack alignment.  */

static unsigned int
ix86_minimum_incoming_stack_boundary (bool sibcall)
{
  unsigned int incoming_stack_boundary;

  /* Stack of interrupt handler is aligned to 128 bits in 64bit
     mode.  */
  if (cfun->machine->func_type != TYPE_NORMAL)
    incoming_stack_boundary = TARGET_64BIT ? 128 : MIN_STACK_BOUNDARY;
  /* Prefer the one specified at command line. */
  else if (ix86_user_incoming_stack_boundary)
    incoming_stack_boundary = ix86_user_incoming_stack_boundary;
  /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
     if -mstackrealign is used, it isn't used for sibcall check and
     estimated stack alignment is 128bit.  */
  else if (!sibcall
	   && ix86_force_align_arg_pointer
	   && crtl->stack_alignment_estimated == 128)
    incoming_stack_boundary = MIN_STACK_BOUNDARY;
  else
    incoming_stack_boundary = ix86_default_incoming_stack_boundary;

  /* Incoming stack alignment can be changed on individual functions
     via force_align_arg_pointer attribute.  We use the smallest
     incoming stack boundary.  */
  if (incoming_stack_boundary > MIN_STACK_BOUNDARY
      && lookup_attribute (ix86_force_align_arg_pointer_string,
			   TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
    incoming_stack_boundary = MIN_STACK_BOUNDARY;

  /* The incoming stack frame has to be aligned at least at
     parm_stack_boundary.  */
  if (incoming_stack_boundary < crtl->parm_stack_boundary)
    incoming_stack_boundary = crtl->parm_stack_boundary;

  /* Stack at entrance of main is aligned by runtime.  We use the
     smallest incoming stack boundary. */
  if (incoming_stack_boundary > MAIN_STACK_BOUNDARY
      && DECL_NAME (current_function_decl)
      && MAIN_NAME_P (DECL_NAME (current_function_decl))
      && DECL_FILE_SCOPE_P (current_function_decl))
    incoming_stack_boundary = MAIN_STACK_BOUNDARY;

  return incoming_stack_boundary;
}

/* Update incoming stack boundary and estimated stack alignment.  */

static void
ix86_update_stack_boundary (void)
{
  ix86_incoming_stack_boundary
    = ix86_minimum_incoming_stack_boundary (false);

  /* x86_64 vararg needs 16byte stack alignment for register save
     area.  */
  if (TARGET_64BIT
      && cfun->stdarg
      && crtl->stack_alignment_estimated < 128)
    crtl->stack_alignment_estimated = 128;

  /* __tls_get_addr needs to be called with 16-byte aligned stack.  */
  if (ix86_tls_descriptor_calls_expanded_in_cfun
      && crtl->preferred_stack_boundary < 128)
    crtl->preferred_stack_boundary = 128;
}

/* Handle the TARGET_GET_DRAP_RTX hook.  Return NULL if no DRAP is
   needed or an rtx for DRAP otherwise.  */

static rtx
ix86_get_drap_rtx (void)
{
  if (ix86_force_drap || !ACCUMULATE_OUTGOING_ARGS)
    crtl->need_drap = true;

  if (stack_realign_drap)
    {
      /* Assign DRAP to vDRAP and returns vDRAP */
      unsigned int regno = find_drap_reg ();
      rtx drap_vreg;
      rtx arg_ptr;
      rtx_insn *seq, *insn;

      arg_ptr = gen_rtx_REG (Pmode, regno);
      crtl->drap_reg = arg_ptr;

      start_sequence ();
      drap_vreg = copy_to_reg (arg_ptr);
      seq = get_insns ();
      end_sequence ();

      insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
      if (!optimize)
	{
	  add_reg_note (insn, REG_CFA_SET_VDRAP, drap_vreg);
	  RTX_FRAME_RELATED_P (insn) = 1;
	}
      return drap_vreg;
    }
  else
    return NULL;
}

/* Handle the TARGET_INTERNAL_ARG_POINTER hook.  */

static rtx
ix86_internal_arg_pointer (void)
{
  return virtual_incoming_args_rtx;
}

struct scratch_reg {
  rtx reg;
  bool saved;
};

/* Return a short-lived scratch register for use on function entry.
   In 32-bit mode, it is valid only after the registers are saved
   in the prologue.  This register must be released by means of
   release_scratch_register_on_entry once it is dead.  */

static void
get_scratch_register_on_entry (struct scratch_reg *sr)
{
  int regno;

  sr->saved = false;

  if (TARGET_64BIT)
    {
      /* We always use R11 in 64-bit mode.  */
      regno = R11_REG;
    }
  else
    {
      tree decl = current_function_decl, fntype = TREE_TYPE (decl);
      bool fastcall_p
	= lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
      bool thiscall_p
	= lookup_attribute ("thiscall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
      bool static_chain_p = DECL_STATIC_CHAIN (decl);
      int regparm = ix86_function_regparm (fntype, decl);
      int drap_regno
	= crtl->drap_reg ? REGNO (crtl->drap_reg) : INVALID_REGNUM;

      /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
	  for the static chain register.  */
      if ((regparm < 1 || (fastcall_p && !static_chain_p))
	  && drap_regno != AX_REG)
	regno = AX_REG;
      /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
	  for the static chain register.  */
      else if (thiscall_p && !static_chain_p && drap_regno != AX_REG)
        regno = AX_REG;
      else if (regparm < 2 && !thiscall_p && drap_regno != DX_REG)
	regno = DX_REG;
      /* ecx is the static chain register.  */
      else if (regparm < 3 && !fastcall_p && !thiscall_p
	       && !static_chain_p
	       && drap_regno != CX_REG)
	regno = CX_REG;
      else if (ix86_save_reg (BX_REG, true))
	regno = BX_REG;
      /* esi is the static chain register.  */
      else if (!(regparm == 3 && static_chain_p)
	       && ix86_save_reg (SI_REG, true))
	regno = SI_REG;
      else if (ix86_save_reg (DI_REG, true))
	regno = DI_REG;
      else
	{
	  regno = (drap_regno == AX_REG ? DX_REG : AX_REG);
	  sr->saved = true;
	}
    }

  sr->reg = gen_rtx_REG (Pmode, regno);
  if (sr->saved)
    {
      rtx_insn *insn = emit_insn (gen_push (sr->reg));
      RTX_FRAME_RELATED_P (insn) = 1;
    }
}

/* Release a scratch register obtained from the preceding function.  */

static void
release_scratch_register_on_entry (struct scratch_reg *sr)
{
  if (sr->saved)
    {
      struct machine_function *m = cfun->machine;
      rtx x, insn = emit_insn (gen_pop (sr->reg));

      /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop.  */
      RTX_FRAME_RELATED_P (insn) = 1;
      x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (UNITS_PER_WORD));
      x = gen_rtx_SET (stack_pointer_rtx, x);
      add_reg_note (insn, REG_FRAME_RELATED_EXPR, x);
      m->fs.sp_offset -= UNITS_PER_WORD;
    }
}

#define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)

/* Emit code to adjust the stack pointer by SIZE bytes while probing it.  */

static void
ix86_adjust_stack_and_probe (const HOST_WIDE_INT size)
{
  /* We skip the probe for the first interval + a small dope of 4 words and
     probe that many bytes past the specified size to maintain a protection
     area at the botton of the stack.  */
  const int dope = 4 * UNITS_PER_WORD;
  rtx size_rtx = GEN_INT (size), last;

  /* See if we have a constant small number of probes to generate.  If so,
     that's the easy case.  The run-time loop is made up of 9 insns in the
     generic case while the compile-time loop is made up of 3+2*(n-1) insns
     for n # of intervals.  */
  if (size <= 4 * PROBE_INTERVAL)
    {
      HOST_WIDE_INT i, adjust;
      bool first_probe = true;

      /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
	 values of N from 1 until it exceeds SIZE.  If only one probe is
	 needed, this will not generate any code.  Then adjust and probe
	 to PROBE_INTERVAL + SIZE.  */
      for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
	{
	  if (first_probe)
	    {
	      adjust = 2 * PROBE_INTERVAL + dope;
	      first_probe = false;
	    }
	  else
	    adjust = PROBE_INTERVAL;

	  emit_insn (gen_rtx_SET (stack_pointer_rtx,
				  plus_constant (Pmode, stack_pointer_rtx,
						 -adjust)));
	  emit_stack_probe (stack_pointer_rtx);
	}

      if (first_probe)
	adjust = size + PROBE_INTERVAL + dope;
      else
        adjust = size + PROBE_INTERVAL - i;

      emit_insn (gen_rtx_SET (stack_pointer_rtx,
			      plus_constant (Pmode, stack_pointer_rtx,
					     -adjust)));
      emit_stack_probe (stack_pointer_rtx);

      /* Adjust back to account for the additional first interval.  */
      last = emit_insn (gen_rtx_SET (stack_pointer_rtx,
				     plus_constant (Pmode, stack_pointer_rtx,
						    PROBE_INTERVAL + dope)));
    }

  /* Otherwise, do the same as above, but in a loop.  Note that we must be
     extra careful with variables wrapping around because we might be at
     the very top (or the very bottom) of the address space and we have
     to be able to handle this case properly; in particular, we use an
     equality test for the loop condition.  */
  else
    {
      HOST_WIDE_INT rounded_size;
      struct scratch_reg sr;

      get_scratch_register_on_entry (&sr);


      /* Step 1: round SIZE to the previous multiple of the interval.  */

      rounded_size = ROUND_DOWN (size, PROBE_INTERVAL);


      /* Step 2: compute initial and final value of the loop counter.  */

      /* SP = SP_0 + PROBE_INTERVAL.  */
      emit_insn (gen_rtx_SET (stack_pointer_rtx,
			      plus_constant (Pmode, stack_pointer_rtx,
					     - (PROBE_INTERVAL + dope))));

      /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE.  */
      if (rounded_size <= (HOST_WIDE_INT_1 << 31))
	emit_insn (gen_rtx_SET (sr.reg,
				plus_constant (Pmode, stack_pointer_rtx,
					       -rounded_size)));
      else
	{
	  emit_move_insn (sr.reg, GEN_INT (-rounded_size));
	  emit_insn (gen_rtx_SET (sr.reg,
				  gen_rtx_PLUS (Pmode, sr.reg,
						stack_pointer_rtx)));
	}


      /* Step 3: the loop

	 do
	   {
	     SP = SP + PROBE_INTERVAL
	     probe at SP
	   }
	 while (SP != LAST_ADDR)

	 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
	 values of N from 1 until it is equal to ROUNDED_SIZE.  */

      emit_insn (ix86_gen_adjust_stack_and_probe (sr.reg, sr.reg, size_rtx));


      /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
	 assert at compile-time that SIZE is equal to ROUNDED_SIZE.  */

      if (size != rounded_size)
	{
	  emit_insn (gen_rtx_SET (stack_pointer_rtx,
			          plus_constant (Pmode, stack_pointer_rtx,
						 rounded_size - size)));
	  emit_stack_probe (stack_pointer_rtx);
	}

      /* Adjust back to account for the additional first interval.  */
      last = emit_insn (gen_rtx_SET (stack_pointer_rtx,
				     plus_constant (Pmode, stack_pointer_rtx,
						    PROBE_INTERVAL + dope)));

      release_scratch_register_on_entry (&sr);
    }

  /* Even if the stack pointer isn't the CFA register, we need to correctly
     describe the adjustments made to it, in particular differentiate the
     frame-related ones from the frame-unrelated ones.  */
  if (size > 0)
    {
      rtx expr = gen_rtx_SEQUENCE (VOIDmode, rtvec_alloc (2));
      XVECEXP (expr, 0, 0)
	= gen_rtx_SET (stack_pointer_rtx,
		       plus_constant (Pmode, stack_pointer_rtx, -size));
      XVECEXP (expr, 0, 1)
	= gen_rtx_SET (stack_pointer_rtx,
		       plus_constant (Pmode, stack_pointer_rtx,
				      PROBE_INTERVAL + dope + size));
      add_reg_note (last, REG_FRAME_RELATED_EXPR, expr);
      RTX_FRAME_RELATED_P (last) = 1;

      cfun->machine->fs.sp_offset += size;
    }

  /* Make sure nothing is scheduled before we are done.  */
  emit_insn (gen_blockage ());
}

/* Adjust the stack pointer up to REG while probing it.  */

const char *
output_adjust_stack_and_probe (rtx reg)
{
  static int labelno = 0;
  char loop_lab[32];
  rtx xops[2];

  ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);

  /* Loop.  */
  ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);

  /* SP = SP + PROBE_INTERVAL.  */
  xops[0] = stack_pointer_rtx;
  xops[1] = GEN_INT (PROBE_INTERVAL);
  output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);

  /* Probe at SP.  */
  xops[1] = const0_rtx;
  output_asm_insn ("or%z0\t{%1, (%0)|DWORD PTR [%0], %1}", xops);

  /* Test if SP == LAST_ADDR.  */
  xops[0] = stack_pointer_rtx;
  xops[1] = reg;
  output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);

  /* Branch.  */
  fputs ("\tjne\t", asm_out_file);
  assemble_name_raw (asm_out_file, loop_lab);
  fputc ('\n', asm_out_file);

  return "";
}

/* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
   inclusive.  These are offsets from the current stack pointer.  */

static void
ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size)
{
  /* See if we have a constant small number of probes to generate.  If so,
     that's the easy case.  The run-time loop is made up of 6 insns in the
     generic case while the compile-time loop is made up of n insns for n #
     of intervals.  */
  if (size <= 6 * PROBE_INTERVAL)
    {
      HOST_WIDE_INT i;

      /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
	 it exceeds SIZE.  If only one probe is needed, this will not
	 generate any code.  Then probe at FIRST + SIZE.  */
      for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
	emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
					 -(first + i)));

      emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
				       -(first + size)));
    }

  /* Otherwise, do the same as above, but in a loop.  Note that we must be
     extra careful with variables wrapping around because we might be at
     the very top (or the very bottom) of the address space and we have
     to be able to handle this case properly; in particular, we use an
     equality test for the loop condition.  */
  else
    {
      HOST_WIDE_INT rounded_size, last;
      struct scratch_reg sr;

      get_scratch_register_on_entry (&sr);


      /* Step 1: round SIZE to the previous multiple of the interval.  */

      rounded_size = ROUND_DOWN (size, PROBE_INTERVAL);


      /* Step 2: compute initial and final value of the loop counter.  */

      /* TEST_OFFSET = FIRST.  */
      emit_move_insn (sr.reg, GEN_INT (-first));

      /* LAST_OFFSET = FIRST + ROUNDED_SIZE.  */
      last = first + rounded_size;


      /* Step 3: the loop

	 do
	   {
	     TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
	     probe at TEST_ADDR
	   }
	 while (TEST_ADDR != LAST_ADDR)

         probes at FIRST + N * PROBE_INTERVAL for values of N from 1
         until it is equal to ROUNDED_SIZE.  */

      emit_insn (ix86_gen_probe_stack_range (sr.reg, sr.reg, GEN_INT (-last)));


      /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
	 that SIZE is equal to ROUNDED_SIZE.  */

      if (size != rounded_size)
	emit_stack_probe (plus_constant (Pmode,
					 gen_rtx_PLUS (Pmode,
						       stack_pointer_rtx,
						       sr.reg),
					 rounded_size - size));

      release_scratch_register_on_entry (&sr);
    }

  /* Make sure nothing is scheduled before we are done.  */
  emit_insn (gen_blockage ());
}

/* Probe a range of stack addresses from REG to END, inclusive.  These are
   offsets from the current stack pointer.  */

const char *
output_probe_stack_range (rtx reg, rtx end)
{
  static int labelno = 0;
  char loop_lab[32];
  rtx xops[3];

  ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);

  /* Loop.  */
  ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);

  /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL.  */
  xops[0] = reg;
  xops[1] = GEN_INT (PROBE_INTERVAL);
  output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);

  /* Probe at TEST_ADDR.  */
  xops[0] = stack_pointer_rtx;
  xops[1] = reg;
  xops[2] = const0_rtx;
  output_asm_insn ("or%z0\t{%2, (%0,%1)|DWORD PTR [%0+%1], %2}", xops);

  /* Test if TEST_ADDR == LAST_ADDR.  */
  xops[0] = reg;
  xops[1] = end;
  output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);

  /* Branch.  */
  fputs ("\tjne\t", asm_out_file);
  assemble_name_raw (asm_out_file, loop_lab);
  fputc ('\n', asm_out_file);

  return "";
}

/* Finalize stack_realign_needed flag, which will guide prologue/epilogue
   to be generated in correct form.  */
static void
ix86_finalize_stack_realign_flags (void)
{
  /* Check if stack realign is really needed after reload, and
     stores result in cfun */
  unsigned int incoming_stack_boundary
    = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
       ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
  unsigned int stack_realign
    = (incoming_stack_boundary
       < (crtl->is_leaf && !ix86_current_function_calls_tls_descriptor
	  ? crtl->max_used_stack_slot_alignment
	  : crtl->stack_alignment_needed));

  if (crtl->stack_realign_finalized)
    {
      /* After stack_realign_needed is finalized, we can't no longer
	 change it.  */
      gcc_assert (crtl->stack_realign_needed == stack_realign);
      return;
    }

  /* If the only reason for frame_pointer_needed is that we conservatively
     assumed stack realignment might be needed, but in the end nothing that
     needed the stack alignment had been spilled, clear frame_pointer_needed
     and say we don't need stack realignment.  */
  if (stack_realign
      && frame_pointer_needed
      && crtl->is_leaf
      && flag_omit_frame_pointer
      && crtl->sp_is_unchanging
      && !ix86_current_function_calls_tls_descriptor
      && !crtl->accesses_prior_frames
      && !cfun->calls_alloca
      && !crtl->calls_eh_return
      /* See ira_setup_eliminable_regset for the rationale.  */
      && !(STACK_CHECK_MOVING_SP
	   && flag_stack_check
	   && flag_exceptions
	   && cfun->can_throw_non_call_exceptions)
      && !ix86_frame_pointer_required ()
      && get_frame_size () == 0
      && ix86_nsaved_sseregs () == 0
      && ix86_varargs_gpr_size + ix86_varargs_fpr_size == 0)
    {
      HARD_REG_SET set_up_by_prologue, prologue_used;
      basic_block bb;

      CLEAR_HARD_REG_SET (prologue_used);
      CLEAR_HARD_REG_SET (set_up_by_prologue);
      add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM);
      add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM);
      add_to_hard_reg_set (&set_up_by_prologue, Pmode,
			   HARD_FRAME_POINTER_REGNUM);
      FOR_EACH_BB_FN (bb, cfun)
        {
          rtx_insn *insn;
	  FOR_BB_INSNS (bb, insn)
	    if (NONDEBUG_INSN_P (insn)
		&& requires_stack_frame_p (insn, prologue_used,
					   set_up_by_prologue))
	      {
		crtl->stack_realign_needed = stack_realign;
		crtl->stack_realign_finalized = true;
		return;
	      }
	}

      /* If drap has been set, but it actually isn't live at the start
	 of the function, there is no reason to set it up.  */
      if (crtl->drap_reg)
	{
	  basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
	  if (! REGNO_REG_SET_P (DF_LR_IN (bb), REGNO (crtl->drap_reg)))
	    {
	      crtl->drap_reg = NULL_RTX;
	      crtl->need_drap = false;
	    }
	}
      else
	cfun->machine->no_drap_save_restore = true;

      frame_pointer_needed = false;
      stack_realign = false;
      crtl->max_used_stack_slot_alignment = incoming_stack_boundary;
      crtl->stack_alignment_needed = incoming_stack_boundary;
      crtl->stack_alignment_estimated = incoming_stack_boundary;
      if (crtl->preferred_stack_boundary > incoming_stack_boundary)
	crtl->preferred_stack_boundary = incoming_stack_boundary;
      df_finish_pass (true);
      df_scan_alloc (NULL);
      df_scan_blocks ();
      df_compute_regs_ever_live (true);
      df_analyze ();
    }

  crtl->stack_realign_needed = stack_realign;
  crtl->stack_realign_finalized = true;
}

/* Delete SET_GOT right after entry block if it is allocated to reg.  */

static void
ix86_elim_entry_set_got (rtx reg)
{
  basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
  rtx_insn *c_insn = BB_HEAD (bb);
  if (!NONDEBUG_INSN_P (c_insn))
    c_insn = next_nonnote_nondebug_insn (c_insn);
  if (c_insn && NONJUMP_INSN_P (c_insn))
    {
      rtx pat = PATTERN (c_insn);
      if (GET_CODE (pat) == PARALLEL)
	{
	  rtx vec = XVECEXP (pat, 0, 0);
	  if (GET_CODE (vec) == SET
	      && XINT (XEXP (vec, 1), 1) == UNSPEC_SET_GOT
	      && REGNO (XEXP (vec, 0)) == REGNO (reg))
	    delete_insn (c_insn);
	}
    }
}

/* Expand the prologue into a bunch of separate insns.  */

void
ix86_expand_prologue (void)
{
  struct machine_function *m = cfun->machine;
  rtx insn, t;
  struct ix86_frame frame;
  HOST_WIDE_INT allocate;
  bool int_registers_saved;
  bool sse_registers_saved;
  rtx static_chain = NULL_RTX;

  ix86_finalize_stack_realign_flags ();

  /* DRAP should not coexist with stack_realign_fp */
  gcc_assert (!(crtl->drap_reg && stack_realign_fp));

  memset (&m->fs, 0, sizeof (m->fs));

  /* Initialize CFA state for before the prologue.  */
  m->fs.cfa_reg = stack_pointer_rtx;
  m->fs.cfa_offset = INCOMING_FRAME_SP_OFFSET;

  /* Track SP offset to the CFA.  We continue tracking this after we've
     swapped the CFA register away from SP.  In the case of re-alignment
     this is fudged; we're interested to offsets within the local frame.  */
  m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
  m->fs.sp_valid = true;

  ix86_compute_frame_layout (&frame);

  if (!TARGET_64BIT && ix86_function_ms_hook_prologue (current_function_decl))
    {
      /* We should have already generated an error for any use of
         ms_hook on a nested function.  */
      gcc_checking_assert (!ix86_static_chain_on_stack);

      /* Check if profiling is active and we shall use profiling before
         prologue variant. If so sorry.  */
      if (crtl->profile && flag_fentry != 0)
        sorry ("ms_hook_prologue attribute isn%'t compatible "
	       "with -mfentry for 32-bit");

      /* In ix86_asm_output_function_label we emitted:
	 8b ff     movl.s %edi,%edi
	 55        push   %ebp
	 8b ec     movl.s %esp,%ebp

	 This matches the hookable function prologue in Win32 API
	 functions in Microsoft Windows XP Service Pack 2 and newer.
	 Wine uses this to enable Windows apps to hook the Win32 API
	 functions provided by Wine.

	 What that means is that we've already set up the frame pointer.  */

      if (frame_pointer_needed
	  && !(crtl->drap_reg && crtl->stack_realign_needed))
	{
	  rtx push, mov;

	  /* We've decided to use the frame pointer already set up.
	     Describe this to the unwinder by pretending that both
	     push and mov insns happen right here.

	     Putting the unwind info here at the end of the ms_hook
	     is done so that we can make absolutely certain we get
	     the required byte sequence at the start of the function,
	     rather than relying on an assembler that can produce
	     the exact encoding required.

	     However it does mean (in the unpatched case) that we have
	     a 1 insn window where the asynchronous unwind info is
	     incorrect.  However, if we placed the unwind info at
	     its correct location we would have incorrect unwind info
	     in the patched case.  Which is probably all moot since
	     I don't expect Wine generates dwarf2 unwind info for the
	     system libraries that use this feature.  */

	  insn = emit_insn (gen_blockage ());

	  push = gen_push (hard_frame_pointer_rtx);
	  mov = gen_rtx_SET (hard_frame_pointer_rtx,
			     stack_pointer_rtx);
	  RTX_FRAME_RELATED_P (push) = 1;
	  RTX_FRAME_RELATED_P (mov) = 1;

	  RTX_FRAME_RELATED_P (insn) = 1;
	  add_reg_note (insn, REG_FRAME_RELATED_EXPR,
			gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, push, mov)));

	  /* Note that gen_push incremented m->fs.cfa_offset, even
	     though we didn't emit the push insn here.  */
	  m->fs.cfa_reg = hard_frame_pointer_rtx;
	  m->fs.fp_offset = m->fs.cfa_offset;
	  m->fs.fp_valid = true;
	}
      else
	{
	  /* The frame pointer is not needed so pop %ebp again.
	     This leaves us with a pristine state.  */
	  emit_insn (gen_pop (hard_frame_pointer_rtx));
	}
    }

  /* The first insn of a function that accepts its static chain on the
     stack is to push the register that would be filled in by a direct
     call.  This insn will be skipped by the trampoline.  */
  else if (ix86_static_chain_on_stack)
    {
      static_chain = ix86_static_chain (cfun->decl, false);
      insn = emit_insn (gen_push (static_chain));
      emit_insn (gen_blockage ());

      /* We don't want to interpret this push insn as a register save,
	 only as a stack adjustment.  The real copy of the register as
	 a save will be done later, if needed.  */
      t = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
      t = gen_rtx_SET (stack_pointer_rtx, t);
      add_reg_note (insn, REG_CFA_ADJUST_CFA, t);
      RTX_FRAME_RELATED_P (insn) = 1;
    }

  /* Emit prologue code to adjust stack alignment and setup DRAP, in case
     of DRAP is needed and stack realignment is really needed after reload */
  if (stack_realign_drap)
    {
      int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;

      /* Can't use DRAP in interrupt function.  */
      if (cfun->machine->func_type != TYPE_NORMAL)
	sorry ("Dynamic Realign Argument Pointer (DRAP) not supported "
	       "in interrupt service routine.  This may be worked "
	       "around by avoiding functions with aggregate return.");

      /* Only need to push parameter pointer reg if it is caller saved.  */
      if (!call_used_regs[REGNO (crtl->drap_reg)])
	{
	  /* Push arg pointer reg */
	  insn = emit_insn (gen_push (crtl->drap_reg));
	  RTX_FRAME_RELATED_P (insn) = 1;
	}

      /* Grab the argument pointer.  */
      t = plus_constant (Pmode, stack_pointer_rtx, m->fs.sp_offset);
      insn = emit_insn (gen_rtx_SET (crtl->drap_reg, t));
      RTX_FRAME_RELATED_P (insn) = 1;
      m->fs.cfa_reg = crtl->drap_reg;
      m->fs.cfa_offset = 0;

      /* Align the stack.  */
      insn = emit_insn (ix86_gen_andsp (stack_pointer_rtx,
					stack_pointer_rtx,
					GEN_INT (-align_bytes)));
      RTX_FRAME_RELATED_P (insn) = 1;

      /* Replicate the return address on the stack so that return
	 address can be reached via (argp - 1) slot.  This is needed
	 to implement macro RETURN_ADDR_RTX and intrinsic function
	 expand_builtin_return_addr etc.  */
      t = plus_constant (Pmode, crtl->drap_reg, -UNITS_PER_WORD);
      t = gen_frame_mem (word_mode, t);
      insn = emit_insn (gen_push (t));
      RTX_FRAME_RELATED_P (insn) = 1;

      /* For the purposes of frame and register save area addressing,
	 we've started over with a new frame.  */
      m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
      m->fs.realigned = true;

      if (static_chain)
	{
	  /* Replicate static chain on the stack so that static chain
	     can be reached via (argp - 2) slot.  This is needed for
	     nested function with stack realignment.  */
	  insn = emit_insn (gen_push (static_chain));
	  RTX_FRAME_RELATED_P (insn) = 1;
	}
    }

  int_registers_saved = (frame.nregs == 0);
  sse_registers_saved = (frame.nsseregs == 0);

  if (frame_pointer_needed && !m->fs.fp_valid)
    {
      /* Note: AT&T enter does NOT have reversed args.  Enter is probably
         slower on all targets.  Also sdb doesn't like it.  */
      insn = emit_insn (gen_push (hard_frame_pointer_rtx));
      RTX_FRAME_RELATED_P (insn) = 1;

      /* Push registers now, before setting the frame pointer
	 on SEH target.  */
      if (!int_registers_saved
	  && TARGET_SEH
	  && !frame.save_regs_using_mov)
	{
	  ix86_emit_save_regs ();
	  int_registers_saved = true;
	  gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
	}

      if (m->fs.sp_offset == frame.hard_frame_pointer_offset)
	{
	  insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
	  RTX_FRAME_RELATED_P (insn) = 1;

	  if (m->fs.cfa_reg == stack_pointer_rtx)
	    m->fs.cfa_reg = hard_frame_pointer_rtx;
	  m->fs.fp_offset = m->fs.sp_offset;
	  m->fs.fp_valid = true;
	}
    }

  if (!int_registers_saved)
    {
      /* If saving registers via PUSH, do so now.  */
      if (!frame.save_regs_using_mov)
	{
	  ix86_emit_save_regs ();
	  int_registers_saved = true;
	  gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
	}

      /* When using red zone we may start register saving before allocating
	 the stack frame saving one cycle of the prologue.  However, avoid
	 doing this if we have to probe the stack; at least on x86_64 the
	 stack probe can turn into a call that clobbers a red zone location. */
      else if (ix86_using_red_zone ()
	       && (! TARGET_STACK_PROBE
		   || frame.stack_pointer_offset < CHECK_STACK_LIMIT))
	{
	  ix86_emit_save_regs_using_mov (frame.reg_save_offset);
	  int_registers_saved = true;
	}
    }

  if (stack_realign_fp)
    {
      int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
      gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT);

      /* The computation of the size of the re-aligned stack frame means
	 that we must allocate the size of the register save area before
	 performing the actual alignment.  Otherwise we cannot guarantee
	 that there's enough storage above the realignment point.  */
      if (m->fs.sp_offset != frame.sse_reg_save_offset)
        pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				   GEN_INT (m->fs.sp_offset
					    - frame.sse_reg_save_offset),
				   -1, false);

      /* Align the stack.  */
      insn = emit_insn (ix86_gen_andsp (stack_pointer_rtx,
					stack_pointer_rtx,
					GEN_INT (-align_bytes)));

      /* For the purposes of register save area addressing, the stack
         pointer is no longer valid.  As for the value of sp_offset,
	 see ix86_compute_frame_layout, which we need to match in order
	 to pass verification of stack_pointer_offset at the end.  */
      m->fs.sp_offset = ROUND_UP (m->fs.sp_offset, align_bytes);
      m->fs.sp_valid = false;
    }

  allocate = frame.stack_pointer_offset - m->fs.sp_offset;

  if (flag_stack_usage_info)
    {
      /* We start to count from ARG_POINTER.  */
      HOST_WIDE_INT stack_size = frame.stack_pointer_offset;

      /* If it was realigned, take into account the fake frame.  */
      if (stack_realign_drap)
	{
	  if (ix86_static_chain_on_stack)
	    stack_size += UNITS_PER_WORD;

	  if (!call_used_regs[REGNO (crtl->drap_reg)])
	    stack_size += UNITS_PER_WORD;

	  /* This over-estimates by 1 minimal-stack-alignment-unit but
	     mitigates that by counting in the new return address slot.  */
	  current_function_dynamic_stack_size
	    += crtl->stack_alignment_needed / BITS_PER_UNIT;
	}

      current_function_static_stack_size = stack_size;
    }

  /* On SEH target with very large frame size, allocate an area to save
     SSE registers (as the very large allocation won't be described).  */
  if (TARGET_SEH
      && frame.stack_pointer_offset > SEH_MAX_FRAME_SIZE
      && !sse_registers_saved)
    {
      HOST_WIDE_INT sse_size =
	frame.sse_reg_save_offset - frame.reg_save_offset;

      gcc_assert (int_registers_saved);

      /* No need to do stack checking as the area will be immediately
	 written.  */
      pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
			         GEN_INT (-sse_size), -1,
				 m->fs.cfa_reg == stack_pointer_rtx);
      allocate -= sse_size;
      ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
      sse_registers_saved = true;
    }

  /* The stack has already been decremented by the instruction calling us
     so probe if the size is non-negative to preserve the protection area.  */
  if (allocate >= 0 && flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
    {
      /* We expect the registers to be saved when probes are used.  */
      gcc_assert (int_registers_saved);

      if (STACK_CHECK_MOVING_SP)
	{
	  if (!(crtl->is_leaf && !cfun->calls_alloca
		&& allocate <= PROBE_INTERVAL))
	    {
	      ix86_adjust_stack_and_probe (allocate);
	      allocate = 0;
	    }
	}
      else
	{
	  HOST_WIDE_INT size = allocate;

	  if (TARGET_64BIT && size >= HOST_WIDE_INT_C (0x80000000))
	    size = 0x80000000 - STACK_CHECK_PROTECT - 1;

	  if (TARGET_STACK_PROBE)
	    {
	      if (crtl->is_leaf && !cfun->calls_alloca)
		{
		  if (size > PROBE_INTERVAL)
		    ix86_emit_probe_stack_range (0, size);
		}
	      else
		ix86_emit_probe_stack_range (0, size + STACK_CHECK_PROTECT);
	    }
	  else
	    {
	      if (crtl->is_leaf && !cfun->calls_alloca)
		{
		  if (size > PROBE_INTERVAL && size > STACK_CHECK_PROTECT)
		    ix86_emit_probe_stack_range (STACK_CHECK_PROTECT,
						 size - STACK_CHECK_PROTECT);
		}
	      else
		ix86_emit_probe_stack_range (STACK_CHECK_PROTECT, size);
	    }
	}
    }

  if (allocate == 0)
    ;
  else if (!ix86_target_stack_probe ()
	   || frame.stack_pointer_offset < CHECK_STACK_LIMIT)
    {
      pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
			         GEN_INT (-allocate), -1,
			         m->fs.cfa_reg == stack_pointer_rtx);
    }
  else
    {
      rtx eax = gen_rtx_REG (Pmode, AX_REG);
      rtx r10 = NULL;
      rtx (*adjust_stack_insn)(rtx, rtx, rtx);
      const bool sp_is_cfa_reg = (m->fs.cfa_reg == stack_pointer_rtx);
      bool eax_live = ix86_eax_live_at_start_p ();
      bool r10_live = false;

      if (TARGET_64BIT)
        r10_live = (DECL_STATIC_CHAIN (current_function_decl) != 0);

      if (eax_live)
	{
	  insn = emit_insn (gen_push (eax));
	  allocate -= UNITS_PER_WORD;
	  /* Note that SEH directives need to continue tracking the stack
	     pointer even after the frame pointer has been set up.  */
	  if (sp_is_cfa_reg || TARGET_SEH)
	    {
	      if (sp_is_cfa_reg)
		m->fs.cfa_offset += UNITS_PER_WORD;
	      RTX_FRAME_RELATED_P (insn) = 1;
	      add_reg_note (insn, REG_FRAME_RELATED_EXPR,
			    gen_rtx_SET (stack_pointer_rtx,
					 plus_constant (Pmode, stack_pointer_rtx,
							-UNITS_PER_WORD)));
	    }
	}

      if (r10_live)
	{
	  r10 = gen_rtx_REG (Pmode, R10_REG);
	  insn = emit_insn (gen_push (r10));
	  allocate -= UNITS_PER_WORD;
	  if (sp_is_cfa_reg || TARGET_SEH)
	    {
	      if (sp_is_cfa_reg)
		m->fs.cfa_offset += UNITS_PER_WORD;
	      RTX_FRAME_RELATED_P (insn) = 1;
	      add_reg_note (insn, REG_FRAME_RELATED_EXPR,
			    gen_rtx_SET (stack_pointer_rtx,
					 plus_constant (Pmode, stack_pointer_rtx,
							-UNITS_PER_WORD)));
	    }
	}

      emit_move_insn (eax, GEN_INT (allocate));
      emit_insn (ix86_gen_allocate_stack_worker (eax, eax));

      /* Use the fact that AX still contains ALLOCATE.  */
      adjust_stack_insn = (Pmode == DImode
			   ? gen_pro_epilogue_adjust_stack_di_sub
			   : gen_pro_epilogue_adjust_stack_si_sub);

      insn = emit_insn (adjust_stack_insn (stack_pointer_rtx,
					   stack_pointer_rtx, eax));

      if (sp_is_cfa_reg || TARGET_SEH)
	{
	  if (sp_is_cfa_reg)
	    m->fs.cfa_offset += allocate;
	  RTX_FRAME_RELATED_P (insn) = 1;
	  add_reg_note (insn, REG_FRAME_RELATED_EXPR,
			gen_rtx_SET (stack_pointer_rtx,
				     plus_constant (Pmode, stack_pointer_rtx,
						    -allocate)));
	}
      m->fs.sp_offset += allocate;

      /* Use stack_pointer_rtx for relative addressing so that code
	 works for realigned stack, too.  */
      if (r10_live && eax_live)
        {
	  t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
	  emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
			  gen_frame_mem (word_mode, t));
	  t = plus_constant (Pmode, t, UNITS_PER_WORD);
	  emit_move_insn (gen_rtx_REG (word_mode, AX_REG),
			  gen_frame_mem (word_mode, t));
	}
      else if (eax_live || r10_live)
	{
	  t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
	  emit_move_insn (gen_rtx_REG (word_mode,
				       (eax_live ? AX_REG : R10_REG)),
			  gen_frame_mem (word_mode, t));
	}
    }
  gcc_assert (m->fs.sp_offset == frame.stack_pointer_offset);

  /* If we havn't already set up the frame pointer, do so now.  */
  if (frame_pointer_needed && !m->fs.fp_valid)
    {
      insn = ix86_gen_add3 (hard_frame_pointer_rtx, stack_pointer_rtx,
			    GEN_INT (frame.stack_pointer_offset
				     - frame.hard_frame_pointer_offset));
      insn = emit_insn (insn);
      RTX_FRAME_RELATED_P (insn) = 1;
      add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);

      if (m->fs.cfa_reg == stack_pointer_rtx)
	m->fs.cfa_reg = hard_frame_pointer_rtx;
      m->fs.fp_offset = frame.hard_frame_pointer_offset;
      m->fs.fp_valid = true;
    }

  if (!int_registers_saved)
    ix86_emit_save_regs_using_mov (frame.reg_save_offset);
  if (!sse_registers_saved)
    ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);

  /* For the mcount profiling on 32 bit PIC mode we need to emit SET_GOT
     in PROLOGUE.  */
  if (!TARGET_64BIT && pic_offset_table_rtx && crtl->profile && !flag_fentry)
    {
      rtx pic = gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM);
      insn = emit_insn (gen_set_got (pic));
      RTX_FRAME_RELATED_P (insn) = 1;
      add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
      emit_insn (gen_prologue_use (pic));
      /* Deleting already emmitted SET_GOT if exist and allocated to
	 REAL_PIC_OFFSET_TABLE_REGNUM.  */
      ix86_elim_entry_set_got (pic);
    }

  if (crtl->drap_reg && !crtl->stack_realign_needed)
    {
      /* vDRAP is setup but after reload it turns out stack realign
         isn't necessary, here we will emit prologue to setup DRAP
         without stack realign adjustment */
      t = choose_baseaddr (0);
      emit_insn (gen_rtx_SET (crtl->drap_reg, t));
    }

  /* Prevent instructions from being scheduled into register save push
     sequence when access to the redzone area is done through frame pointer.
     The offset between the frame pointer and the stack pointer is calculated
     relative to the value of the stack pointer at the end of the function
     prologue, and moving instructions that access redzone area via frame
     pointer inside push sequence violates this assumption.  */
  if (frame_pointer_needed && frame.red_zone_size)
    emit_insn (gen_memory_blockage ());

  /* SEH requires that the prologue end within 256 bytes of the start of
     the function.  Prevent instruction schedules that would extend that.
     Further, prevent alloca modifications to the stack pointer from being
     combined with prologue modifications.  */
  if (TARGET_SEH)
    emit_insn (gen_prologue_use (stack_pointer_rtx));
}

/* Emit code to restore REG using a POP insn.  */

static void
ix86_emit_restore_reg_using_pop (rtx reg)
{
  struct machine_function *m = cfun->machine;
  rtx_insn *insn = emit_insn (gen_pop (reg));

  ix86_add_cfa_restore_note (insn, reg, m->fs.sp_offset);
  m->fs.sp_offset -= UNITS_PER_WORD;

  if (m->fs.cfa_reg == crtl->drap_reg
      && REGNO (reg) == REGNO (crtl->drap_reg))
    {
      /* Previously we'd represented the CFA as an expression
	 like *(%ebp - 8).  We've just popped that value from
	 the stack, which means we need to reset the CFA to
	 the drap register.  This will remain until we restore
	 the stack pointer.  */
      add_reg_note (insn, REG_CFA_DEF_CFA, reg);
      RTX_FRAME_RELATED_P (insn) = 1;

      /* This means that the DRAP register is valid for addressing too.  */
      m->fs.drap_valid = true;
      return;
    }

  if (m->fs.cfa_reg == stack_pointer_rtx)
    {
      rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
      x = gen_rtx_SET (stack_pointer_rtx, x);
      add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
      RTX_FRAME_RELATED_P (insn) = 1;

      m->fs.cfa_offset -= UNITS_PER_WORD;
    }

  /* When the frame pointer is the CFA, and we pop it, we are
     swapping back to the stack pointer as the CFA.  This happens
     for stack frames that don't allocate other data, so we assume
     the stack pointer is now pointing at the return address, i.e.
     the function entry state, which makes the offset be 1 word.  */
  if (reg == hard_frame_pointer_rtx)
    {
      m->fs.fp_valid = false;
      if (m->fs.cfa_reg == hard_frame_pointer_rtx)
	{
	  m->fs.cfa_reg = stack_pointer_rtx;
	  m->fs.cfa_offset -= UNITS_PER_WORD;

	  add_reg_note (insn, REG_CFA_DEF_CFA,
			gen_rtx_PLUS (Pmode, stack_pointer_rtx,
				      GEN_INT (m->fs.cfa_offset)));
	  RTX_FRAME_RELATED_P (insn) = 1;
	}
    }
}

/* Emit code to restore saved registers using POP insns.  */

static void
ix86_emit_restore_regs_using_pop (void)
{
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, false))
      ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno));
}

/* Emit code and notes for the LEAVE instruction.  */

static void
ix86_emit_leave (void)
{
  struct machine_function *m = cfun->machine;
  rtx_insn *insn = emit_insn (ix86_gen_leave ());

  ix86_add_queued_cfa_restore_notes (insn);

  gcc_assert (m->fs.fp_valid);
  m->fs.sp_valid = true;
  m->fs.sp_offset = m->fs.fp_offset - UNITS_PER_WORD;
  m->fs.fp_valid = false;

  if (m->fs.cfa_reg == hard_frame_pointer_rtx)
    {
      m->fs.cfa_reg = stack_pointer_rtx;
      m->fs.cfa_offset = m->fs.sp_offset;

      add_reg_note (insn, REG_CFA_DEF_CFA,
		    plus_constant (Pmode, stack_pointer_rtx,
				   m->fs.sp_offset));
      RTX_FRAME_RELATED_P (insn) = 1;
    }
  ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx,
			     m->fs.fp_offset);
}

/* Emit code to restore saved registers using MOV insns.
   First register is restored from CFA - CFA_OFFSET.  */
static void
ix86_emit_restore_regs_using_mov (HOST_WIDE_INT cfa_offset,
				  bool maybe_eh_return)
{
  struct machine_function *m = cfun->machine;
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
      {
	rtx reg = gen_rtx_REG (word_mode, regno);
	rtx mem;
	rtx_insn *insn;

	mem = choose_baseaddr (cfa_offset);
	mem = gen_frame_mem (word_mode, mem);
	insn = emit_move_insn (reg, mem);

        if (m->fs.cfa_reg == crtl->drap_reg && regno == REGNO (crtl->drap_reg))
	  {
	    /* Previously we'd represented the CFA as an expression
	       like *(%ebp - 8).  We've just popped that value from
	       the stack, which means we need to reset the CFA to
	       the drap register.  This will remain until we restore
	       the stack pointer.  */
	    add_reg_note (insn, REG_CFA_DEF_CFA, reg);
	    RTX_FRAME_RELATED_P (insn) = 1;

	    /* This means that the DRAP register is valid for addressing.  */
	    m->fs.drap_valid = true;
	  }
	else
	  ix86_add_cfa_restore_note (NULL, reg, cfa_offset);

	cfa_offset -= UNITS_PER_WORD;
      }
}

/* Emit code to restore saved registers using MOV insns.
   First register is restored from CFA - CFA_OFFSET.  */
static void
ix86_emit_restore_sse_regs_using_mov (HOST_WIDE_INT cfa_offset,
				      bool maybe_eh_return)
{
  unsigned int regno;

  for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
      {
	rtx reg = gen_rtx_REG (V4SFmode, regno);
	rtx mem;
	unsigned int align;

	mem = choose_baseaddr (cfa_offset);
	mem = gen_rtx_MEM (V4SFmode, mem);

	/* The location is aligned up to INCOMING_STACK_BOUNDARY.  */
	align = MIN (GET_MODE_ALIGNMENT (V4SFmode), INCOMING_STACK_BOUNDARY);
	set_mem_align (mem, align);
	emit_insn (gen_rtx_SET (reg, mem));

	ix86_add_cfa_restore_note (NULL, reg, cfa_offset);

	cfa_offset -= GET_MODE_SIZE (V4SFmode);
      }
}

/* Restore function stack, frame, and registers.  */

void
ix86_expand_epilogue (int style)
{
  struct machine_function *m = cfun->machine;
  struct machine_frame_state frame_state_save = m->fs;
  struct ix86_frame frame;
  bool restore_regs_via_mov;
  bool using_drap;

  ix86_finalize_stack_realign_flags ();
  ix86_compute_frame_layout (&frame);

  m->fs.sp_valid = (!frame_pointer_needed
		    || (crtl->sp_is_unchanging
			&& !stack_realign_fp));
  gcc_assert (!m->fs.sp_valid
	      || m->fs.sp_offset == frame.stack_pointer_offset);

  /* The FP must be valid if the frame pointer is present.  */
  gcc_assert (frame_pointer_needed == m->fs.fp_valid);
  gcc_assert (!m->fs.fp_valid
	      || m->fs.fp_offset == frame.hard_frame_pointer_offset);

  /* We must have *some* valid pointer to the stack frame.  */
  gcc_assert (m->fs.sp_valid || m->fs.fp_valid);

  /* The DRAP is never valid at this point.  */
  gcc_assert (!m->fs.drap_valid);

  /* See the comment about red zone and frame
     pointer usage in ix86_expand_prologue.  */
  if (frame_pointer_needed && frame.red_zone_size)
    emit_insn (gen_memory_blockage ());

  using_drap = crtl->drap_reg && crtl->stack_realign_needed;
  gcc_assert (!using_drap || m->fs.cfa_reg == crtl->drap_reg);

  /* Determine the CFA offset of the end of the red-zone.  */
  m->fs.red_zone_offset = 0;
  if (ix86_using_red_zone () && crtl->args.pops_args < 65536)
    {
      /* The red-zone begins below the return address.  */
      m->fs.red_zone_offset = RED_ZONE_SIZE + UNITS_PER_WORD;

      /* When the register save area is in the aligned portion of
         the stack, determine the maximum runtime displacement that
	 matches up with the aligned frame.  */
      if (stack_realign_drap)
	m->fs.red_zone_offset -= (crtl->stack_alignment_needed / BITS_PER_UNIT
				  + UNITS_PER_WORD);
    }

  /* Special care must be taken for the normal return case of a function
     using eh_return: the eax and edx registers are marked as saved, but
     not restored along this path.  Adjust the save location to match.  */
  if (crtl->calls_eh_return && style != 2)
    frame.reg_save_offset -= 2 * UNITS_PER_WORD;

  /* EH_RETURN requires the use of moves to function properly.  */
  if (crtl->calls_eh_return)
    restore_regs_via_mov = true;
  /* SEH requires the use of pops to identify the epilogue.  */
  else if (TARGET_SEH)
    restore_regs_via_mov = false;
  /* If we're only restoring one register and sp is not valid then
     using a move instruction to restore the register since it's
     less work than reloading sp and popping the register.  */
  else if (!m->fs.sp_valid && frame.nregs <= 1)
    restore_regs_via_mov = true;
  else if (TARGET_EPILOGUE_USING_MOVE
	   && cfun->machine->use_fast_prologue_epilogue
	   && (frame.nregs > 1
	       || m->fs.sp_offset != frame.reg_save_offset))
    restore_regs_via_mov = true;
  else if (frame_pointer_needed
	   && !frame.nregs
	   && m->fs.sp_offset != frame.reg_save_offset)
    restore_regs_via_mov = true;
  else if (frame_pointer_needed
	   && TARGET_USE_LEAVE
	   && cfun->machine->use_fast_prologue_epilogue
	   && frame.nregs == 1)
    restore_regs_via_mov = true;
  else
    restore_regs_via_mov = false;

  if (restore_regs_via_mov || frame.nsseregs)
    {
      /* Ensure that the entire register save area is addressable via
	 the stack pointer, if we will restore via sp.  */
      if (TARGET_64BIT
	  && m->fs.sp_offset > 0x7fffffff
	  && !(m->fs.fp_valid || m->fs.drap_valid)
	  && (frame.nsseregs + frame.nregs) != 0)
	{
	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				     GEN_INT (m->fs.sp_offset
					      - frame.sse_reg_save_offset),
				     style,
				     m->fs.cfa_reg == stack_pointer_rtx);
	}
    }

  /* If there are any SSE registers to restore, then we have to do it
     via moves, since there's obviously no pop for SSE regs.  */
  if (frame.nsseregs)
    ix86_emit_restore_sse_regs_using_mov (frame.sse_reg_save_offset,
					  style == 2);

  if (restore_regs_via_mov)
    {
      rtx t;

      if (frame.nregs)
	ix86_emit_restore_regs_using_mov (frame.reg_save_offset, style == 2);

      /* eh_return epilogues need %ecx added to the stack pointer.  */
      if (style == 2)
	{
	  rtx sa = EH_RETURN_STACKADJ_RTX;
	  rtx_insn *insn;

	  /* %ecx can't be used for both DRAP register and eh_return.  */
	  if (crtl->drap_reg)
	    gcc_assert (REGNO (crtl->drap_reg) != CX_REG);

	  /* regparm nested functions don't work with eh_return.  */
	  gcc_assert (!ix86_static_chain_on_stack);

	  if (frame_pointer_needed)
	    {
	      t = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
	      t = plus_constant (Pmode, t, m->fs.fp_offset - UNITS_PER_WORD);
	      emit_insn (gen_rtx_SET (sa, t));

	      t = gen_frame_mem (Pmode, hard_frame_pointer_rtx);
	      insn = emit_move_insn (hard_frame_pointer_rtx, t);

	      /* Note that we use SA as a temporary CFA, as the return
		 address is at the proper place relative to it.  We
		 pretend this happens at the FP restore insn because
		 prior to this insn the FP would be stored at the wrong
		 offset relative to SA, and after this insn we have no
		 other reasonable register to use for the CFA.  We don't
		 bother resetting the CFA to the SP for the duration of
		 the return insn.  */
	      add_reg_note (insn, REG_CFA_DEF_CFA,
			    plus_constant (Pmode, sa, UNITS_PER_WORD));
	      ix86_add_queued_cfa_restore_notes (insn);
	      add_reg_note (insn, REG_CFA_RESTORE, hard_frame_pointer_rtx);
	      RTX_FRAME_RELATED_P (insn) = 1;

	      m->fs.cfa_reg = sa;
	      m->fs.cfa_offset = UNITS_PER_WORD;
	      m->fs.fp_valid = false;

	      pro_epilogue_adjust_stack (stack_pointer_rtx, sa,
					 const0_rtx, style, false);
	    }
	  else
	    {
	      t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
	      t = plus_constant (Pmode, t, m->fs.sp_offset - UNITS_PER_WORD);
	      insn = emit_insn (gen_rtx_SET (stack_pointer_rtx, t));
	      ix86_add_queued_cfa_restore_notes (insn);

	      gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
	      if (m->fs.cfa_offset != UNITS_PER_WORD)
		{
		  m->fs.cfa_offset = UNITS_PER_WORD;
		  add_reg_note (insn, REG_CFA_DEF_CFA,
				plus_constant (Pmode, stack_pointer_rtx,
					       UNITS_PER_WORD));
		  RTX_FRAME_RELATED_P (insn) = 1;
		}
	    }
	  m->fs.sp_offset = UNITS_PER_WORD;
	  m->fs.sp_valid = true;
	}
    }
  else
    {
      /* SEH requires that the function end with (1) a stack adjustment
	 if necessary, (2) a sequence of pops, and (3) a return or
	 jump instruction.  Prevent insns from the function body from
	 being scheduled into this sequence.  */
      if (TARGET_SEH)
	{
	  /* Prevent a catch region from being adjacent to the standard
	     epilogue sequence.  Unfortuantely crtl->uses_eh_lsda nor
	     several other flags that would be interesting to test are
	     not yet set up.  */
	  if (flag_non_call_exceptions)
	    emit_insn (gen_nops (const1_rtx));
	  else
	    emit_insn (gen_blockage ());
	}

      /* First step is to deallocate the stack frame so that we can
	 pop the registers.  Also do it on SEH target for very large
	 frame as the emitted instructions aren't allowed by the ABI in
	 epilogues.  */
      if (!m->fs.sp_valid
 	  || (TARGET_SEH
	      && (m->fs.sp_offset - frame.reg_save_offset
		  >= SEH_MAX_FRAME_SIZE)))
	{
	  pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx,
				     GEN_INT (m->fs.fp_offset
					      - frame.reg_save_offset),
				     style, false);
	}
      else if (m->fs.sp_offset != frame.reg_save_offset)
	{
	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				     GEN_INT (m->fs.sp_offset
					      - frame.reg_save_offset),
				     style,
				     m->fs.cfa_reg == stack_pointer_rtx);
	}

      ix86_emit_restore_regs_using_pop ();
    }

  /* If we used a stack pointer and haven't already got rid of it,
     then do so now.  */
  if (m->fs.fp_valid)
    {
      /* If the stack pointer is valid and pointing at the frame
	 pointer store address, then we only need a pop.  */
      if (m->fs.sp_valid && m->fs.sp_offset == frame.hfp_save_offset)
	ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
      /* Leave results in shorter dependency chains on CPUs that are
	 able to grok it fast.  */
      else if (TARGET_USE_LEAVE
	       || optimize_bb_for_size_p (EXIT_BLOCK_PTR_FOR_FN (cfun))
	       || !cfun->machine->use_fast_prologue_epilogue)
	ix86_emit_leave ();
      else
        {
	  pro_epilogue_adjust_stack (stack_pointer_rtx,
				     hard_frame_pointer_rtx,
				     const0_rtx, style, !using_drap);
	  ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
        }
    }

  if (using_drap)
    {
      int param_ptr_offset = UNITS_PER_WORD;
      rtx_insn *insn;

      gcc_assert (stack_realign_drap);

      if (ix86_static_chain_on_stack)
	param_ptr_offset += UNITS_PER_WORD;
      if (!call_used_regs[REGNO (crtl->drap_reg)])
	param_ptr_offset += UNITS_PER_WORD;

      insn = emit_insn (gen_rtx_SET
			(stack_pointer_rtx,
			 gen_rtx_PLUS (Pmode,
				       crtl->drap_reg,
				       GEN_INT (-param_ptr_offset))));
      m->fs.cfa_reg = stack_pointer_rtx;
      m->fs.cfa_offset = param_ptr_offset;
      m->fs.sp_offset = param_ptr_offset;
      m->fs.realigned = false;

      add_reg_note (insn, REG_CFA_DEF_CFA,
		    gen_rtx_PLUS (Pmode, stack_pointer_rtx,
				  GEN_INT (param_ptr_offset)));
      RTX_FRAME_RELATED_P (insn) = 1;

      if (!call_used_regs[REGNO (crtl->drap_reg)])
	ix86_emit_restore_reg_using_pop (crtl->drap_reg);
    }

  /* At this point the stack pointer must be valid, and we must have
     restored all of the registers.  We may not have deallocated the
     entire stack frame.  We've delayed this until now because it may
     be possible to merge the local stack deallocation with the
     deallocation forced by ix86_static_chain_on_stack.   */
  gcc_assert (m->fs.sp_valid);
  gcc_assert (!m->fs.fp_valid);
  gcc_assert (!m->fs.realigned);
  if (m->fs.sp_offset != UNITS_PER_WORD)
    {
      pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				 GEN_INT (m->fs.sp_offset - UNITS_PER_WORD),
				 style, true);
    }
  else
    ix86_add_queued_cfa_restore_notes (get_last_insn ());

  /* Sibcall epilogues don't want a return instruction.  */
  if (style == 0)
    {
      m->fs = frame_state_save;
      return;
    }

  if (cfun->machine->func_type != TYPE_NORMAL)
    {
      /* Return with the "IRET" instruction from interrupt handler.
	 Pop the 'ERROR_CODE' off the stack before the 'IRET'
	 instruction in exception handler.  */
      if (cfun->machine->func_type == TYPE_EXCEPTION)
	{
	  rtx r = plus_constant (Pmode, stack_pointer_rtx,
				 UNITS_PER_WORD);
	  emit_insn (gen_rtx_SET (stack_pointer_rtx, r));
	}
      emit_jump_insn (gen_interrupt_return ());
    }
  else if (crtl->args.pops_args && crtl->args.size)
    {
      rtx popc = GEN_INT (crtl->args.pops_args);

      /* i386 can only pop 64K bytes.  If asked to pop more, pop return
	 address, do explicit add, and jump indirectly to the caller.  */

      if (crtl->args.pops_args >= 65536)
	{
	  rtx ecx = gen_rtx_REG (SImode, CX_REG);
	  rtx_insn *insn;

	  /* There is no "pascal" calling convention in any 64bit ABI.  */
	  gcc_assert (!TARGET_64BIT);

	  insn = emit_insn (gen_pop (ecx));
	  m->fs.cfa_offset -= UNITS_PER_WORD;
	  m->fs.sp_offset -= UNITS_PER_WORD;

	  rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
	  x = gen_rtx_SET (stack_pointer_rtx, x);
	  add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
	  add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
	  RTX_FRAME_RELATED_P (insn) = 1;

	  pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
				     popc, -1, true);
	  emit_jump_insn (gen_simple_return_indirect_internal (ecx));
	}
      else
	emit_jump_insn (gen_simple_return_pop_internal (popc));
    }
  else
    emit_jump_insn (gen_simple_return_internal ());

  /* Restore the state back to the state from the prologue,
     so that it's correct for the next epilogue.  */
  m->fs = frame_state_save;
}

/* Reset from the function's potential modifications.  */

static void
ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED, HOST_WIDE_INT)
{
  if (pic_offset_table_rtx
      && !ix86_use_pseudo_pic_reg ())
    SET_REGNO (pic_offset_table_rtx, REAL_PIC_OFFSET_TABLE_REGNUM);

  if (TARGET_MACHO)
    {
      rtx_insn *insn = get_last_insn ();
      rtx_insn *deleted_debug_label = NULL;

      /* Mach-O doesn't support labels at the end of objects, so if
         it looks like we might want one, take special action.
        First, collect any sequence of deleted debug labels.  */
      while (insn
	     && NOTE_P (insn)
	     && NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
	{
	  /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
	     notes only, instead set their CODE_LABEL_NUMBER to -1,
	     otherwise there would be code generation differences
	     in between -g and -g0.  */
	  if (NOTE_P (insn) && NOTE_KIND (insn)
	      == NOTE_INSN_DELETED_DEBUG_LABEL)
	    deleted_debug_label = insn;
	  insn = PREV_INSN (insn);
	}

      /* If we have:
	 label:
	    barrier
	  then this needs to be detected, so skip past the barrier.  */

      if (insn && BARRIER_P (insn))
	insn = PREV_INSN (insn);

      /* Up to now we've only seen notes or barriers.  */
      if (insn)
	{
	  if (LABEL_P (insn)
	      || (NOTE_P (insn)
		  && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL))
	    /* Trailing label.  */
	    fputs ("\tnop\n", file);
	  else if (cfun && ! cfun->is_thunk)
	    {
	      /* See if we have a completely empty function body, skipping
	         the special case of the picbase thunk emitted as asm.  */
	      while (insn && ! INSN_P (insn))
		insn = PREV_INSN (insn);
	      /* If we don't find any insns, we've got an empty function body;
		 I.e. completely empty - without a return or branch.  This is
		 taken as the case where a function body has been removed
		 because it contains an inline __builtin_unreachable().  GCC
		 declares that reaching __builtin_unreachable() means UB so
		 we're not obliged to do anything special; however, we want
		 non-zero-sized function bodies.  To meet this, and help the
		 user out, let's trap the case.  */
	      if (insn == NULL)
		fputs ("\tud2\n", file);
	    }
	}
      else if (deleted_debug_label)
	for (insn = deleted_debug_label; insn; insn = NEXT_INSN (insn))
	  if (NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
	    CODE_LABEL_NUMBER (insn) = -1;
    }
}

/* Return a scratch register to use in the split stack prologue.  The
   split stack prologue is used for -fsplit-stack.  It is the first
   instructions in the function, even before the regular prologue.
   The scratch register can be any caller-saved register which is not
   used for parameters or for the static chain.  */

static unsigned int
split_stack_prologue_scratch_regno (void)
{
  if (TARGET_64BIT)
    return R11_REG;
  else
    {
      bool is_fastcall, is_thiscall;
      int regparm;

      is_fastcall = (lookup_attribute ("fastcall",
				       TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
		     != NULL);
      is_thiscall = (lookup_attribute ("thiscall",
				       TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
		     != NULL);
      regparm = ix86_function_regparm (TREE_TYPE (cfun->decl), cfun->decl);

      if (is_fastcall)
	{
	  if (DECL_STATIC_CHAIN (cfun->decl))
	    {
	      sorry ("-fsplit-stack does not support fastcall with "
		     "nested function");
	      return INVALID_REGNUM;
	    }
	  return AX_REG;
	}
      else if (is_thiscall)
        {
	  if (!DECL_STATIC_CHAIN (cfun->decl))
	    return DX_REG;
	  return AX_REG;
	}
      else if (regparm < 3)
	{
	  if (!DECL_STATIC_CHAIN (cfun->decl))
	    return CX_REG;
	  else
	    {
	      if (regparm >= 2)
		{
		  sorry ("-fsplit-stack does not support 2 register "
			 "parameters for a nested function");
		  return INVALID_REGNUM;
		}
	      return DX_REG;
	    }
	}
      else
	{
	  /* FIXME: We could make this work by pushing a register
	     around the addition and comparison.  */
	  sorry ("-fsplit-stack does not support 3 register parameters");
	  return INVALID_REGNUM;
	}
    }
}

/* A SYMBOL_REF for the function which allocates new stackspace for
   -fsplit-stack.  */

static GTY(()) rtx split_stack_fn;

/* A SYMBOL_REF for the more stack function when using the large
   model.  */

static GTY(()) rtx split_stack_fn_large;

/* Handle -fsplit-stack.  These are the first instructions in the
   function, even before the regular prologue.  */

void
ix86_expand_split_stack_prologue (void)
{
  struct ix86_frame frame;
  HOST_WIDE_INT allocate;
  unsigned HOST_WIDE_INT args_size;
  rtx_code_label *label;
  rtx limit, current, allocate_rtx, call_insn, call_fusage;
  rtx scratch_reg = NULL_RTX;
  rtx_code_label *varargs_label = NULL;
  rtx fn;

  gcc_assert (flag_split_stack && reload_completed);

  ix86_finalize_stack_realign_flags ();
  ix86_compute_frame_layout (&frame);
  allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET;

  /* This is the label we will branch to if we have enough stack
     space.  We expect the basic block reordering pass to reverse this
     branch if optimizing, so that we branch in the unlikely case.  */
  label = gen_label_rtx ();

  /* We need to compare the stack pointer minus the frame size with
     the stack boundary in the TCB.  The stack boundary always gives
     us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
     can compare directly.  Otherwise we need to do an addition.  */

  limit = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
			  UNSPEC_STACK_CHECK);
  limit = gen_rtx_CONST (Pmode, limit);
  limit = gen_rtx_MEM (Pmode, limit);
  if (allocate < SPLIT_STACK_AVAILABLE)
    current = stack_pointer_rtx;
  else
    {
      unsigned int scratch_regno;
      rtx offset;

      /* We need a scratch register to hold the stack pointer minus
	 the required frame size.  Since this is the very start of the
	 function, the scratch register can be any caller-saved
	 register which is not used for parameters.  */
      offset = GEN_INT (- allocate);
      scratch_regno = split_stack_prologue_scratch_regno ();
      if (scratch_regno == INVALID_REGNUM)
	return;
      scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
      if (!TARGET_64BIT || x86_64_immediate_operand (offset, Pmode))
	{
	  /* We don't use ix86_gen_add3 in this case because it will
	     want to split to lea, but when not optimizing the insn
	     will not be split after this point.  */
	  emit_insn (gen_rtx_SET (scratch_reg,
				  gen_rtx_PLUS (Pmode, stack_pointer_rtx,
						offset)));
	}
      else
	{
	  emit_move_insn (scratch_reg, offset);
	  emit_insn (ix86_gen_add3 (scratch_reg, scratch_reg,
				    stack_pointer_rtx));
	}
      current = scratch_reg;
    }

  ix86_expand_branch (GEU, current, limit, label);
  rtx_insn *jump_insn = get_last_insn ();
  JUMP_LABEL (jump_insn) = label;

  /* Mark the jump as very likely to be taken.  */
  add_int_reg_note (jump_insn, REG_BR_PROB,
		    REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100);

  if (split_stack_fn == NULL_RTX)
    {
      split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack");
      SYMBOL_REF_FLAGS (split_stack_fn) |= SYMBOL_FLAG_LOCAL;
    }
  fn = split_stack_fn;

  /* Get more stack space.  We pass in the desired stack space and the
     size of the arguments to copy to the new stack.  In 32-bit mode
     we push the parameters; __morestack will return on a new stack
     anyhow.  In 64-bit mode we pass the parameters in r10 and
     r11.  */
  allocate_rtx = GEN_INT (allocate);
  args_size = crtl->args.size >= 0 ? crtl->args.size : 0;
  call_fusage = NULL_RTX;
  rtx pop = NULL_RTX;
  if (TARGET_64BIT)
    {
      rtx reg10, reg11;

      reg10 = gen_rtx_REG (Pmode, R10_REG);
      reg11 = gen_rtx_REG (Pmode, R11_REG);

      /* If this function uses a static chain, it will be in %r10.
	 Preserve it across the call to __morestack.  */
      if (DECL_STATIC_CHAIN (cfun->decl))
	{
	  rtx rax;

	  rax = gen_rtx_REG (word_mode, AX_REG);
	  emit_move_insn (rax, gen_rtx_REG (word_mode, R10_REG));
	  use_reg (&call_fusage, rax);
	}

      if ((ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
          && !TARGET_PECOFF)
	{
	  HOST_WIDE_INT argval;

	  gcc_assert (Pmode == DImode);
	  /* When using the large model we need to load the address
	     into a register, and we've run out of registers.  So we
	     switch to a different calling convention, and we call a
	     different function: __morestack_large.  We pass the
	     argument size in the upper 32 bits of r10 and pass the
	     frame size in the lower 32 bits.  */
	  gcc_assert ((allocate & HOST_WIDE_INT_C (0xffffffff)) == allocate);
	  gcc_assert ((args_size & 0xffffffff) == args_size);

	  if (split_stack_fn_large == NULL_RTX)
	    {
	      split_stack_fn_large =
	        gen_rtx_SYMBOL_REF (Pmode, "__morestack_large_model");
	      SYMBOL_REF_FLAGS (split_stack_fn_large) |= SYMBOL_FLAG_LOCAL;
	    }
	  if (ix86_cmodel == CM_LARGE_PIC)
	    {
	      rtx_code_label *label;
	      rtx x;

	      label = gen_label_rtx ();
	      emit_label (label);
	      LABEL_PRESERVE_P (label) = 1;
	      emit_insn (gen_set_rip_rex64 (reg10, label));
	      emit_insn (gen_set_got_offset_rex64 (reg11, label));
	      emit_insn (ix86_gen_add3 (reg10, reg10, reg11));
	      x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, split_stack_fn_large),
				  UNSPEC_GOT);
	      x = gen_rtx_CONST (Pmode, x);
	      emit_move_insn (reg11, x);
	      x = gen_rtx_PLUS (Pmode, reg10, reg11);
	      x = gen_const_mem (Pmode, x);
	      emit_move_insn (reg11, x);
	    }
	  else
	    emit_move_insn (reg11, split_stack_fn_large);

	  fn = reg11;

	  argval = ((args_size << 16) << 16) + allocate;
	  emit_move_insn (reg10, GEN_INT (argval));
	}
      else
	{
	  emit_move_insn (reg10, allocate_rtx);
	  emit_move_insn (reg11, GEN_INT (args_size));
	  use_reg (&call_fusage, reg11);
	}

      use_reg (&call_fusage, reg10);
    }
  else
    {
      rtx_insn *insn = emit_insn (gen_push (GEN_INT (args_size)));
      add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (UNITS_PER_WORD));
      insn = emit_insn (gen_push (allocate_rtx));
      add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (2 * UNITS_PER_WORD));
      pop = GEN_INT (2 * UNITS_PER_WORD);
    }
  call_insn = ix86_expand_call (NULL_RTX, gen_rtx_MEM (QImode, fn),
				GEN_INT (UNITS_PER_WORD), constm1_rtx,
				pop, false);
  add_function_usage_to (call_insn, call_fusage);
  if (!TARGET_64BIT)
    add_reg_note (call_insn, REG_ARGS_SIZE, GEN_INT (0));
  /* Indicate that this function can't jump to non-local gotos.  */
  make_reg_eh_region_note_nothrow_nononlocal (as_a <rtx_insn *> (call_insn));

  /* In order to make call/return prediction work right, we now need
     to execute a return instruction.  See
     libgcc/config/i386/morestack.S for the details on how this works.

     For flow purposes gcc must not see this as a return
     instruction--we need control flow to continue at the subsequent
     label.  Therefore, we use an unspec.  */
  gcc_assert (crtl->args.pops_args < 65536);
  emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args)));

  /* If we are in 64-bit mode and this function uses a static chain,
     we saved %r10 in %rax before calling _morestack.  */
  if (TARGET_64BIT && DECL_STATIC_CHAIN (cfun->decl))
    emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
		    gen_rtx_REG (word_mode, AX_REG));

  /* If this function calls va_start, we need to store a pointer to
     the arguments on the old stack, because they may not have been
     all copied to the new stack.  At this point the old stack can be
     found at the frame pointer value used by __morestack, because
     __morestack has set that up before calling back to us.  Here we
     store that pointer in a scratch register, and in
     ix86_expand_prologue we store the scratch register in a stack
     slot.  */
  if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
    {
      unsigned int scratch_regno;
      rtx frame_reg;
      int words;

      scratch_regno = split_stack_prologue_scratch_regno ();
      scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
      frame_reg = gen_rtx_REG (Pmode, BP_REG);

      /* 64-bit:
	 fp -> old fp value
	       return address within this function
	       return address of caller of this function
	       stack arguments
	 So we add three words to get to the stack arguments.

	 32-bit:
	 fp -> old fp value
	       return address within this function
               first argument to __morestack
               second argument to __morestack
               return address of caller of this function
               stack arguments
         So we add five words to get to the stack arguments.
      */
      words = TARGET_64BIT ? 3 : 5;
      emit_insn (gen_rtx_SET (scratch_reg,
			      gen_rtx_PLUS (Pmode, frame_reg,
					    GEN_INT (words * UNITS_PER_WORD))));

      varargs_label = gen_label_rtx ();
      emit_jump_insn (gen_jump (varargs_label));
      JUMP_LABEL (get_last_insn ()) = varargs_label;

      emit_barrier ();
    }

  emit_label (label);
  LABEL_NUSES (label) = 1;

  /* If this function calls va_start, we now have to set the scratch
     register for the case where we do not call __morestack.  In this
     case we need to set it based on the stack pointer.  */
  if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
    {
      emit_insn (gen_rtx_SET (scratch_reg,
			      gen_rtx_PLUS (Pmode, stack_pointer_rtx,
					    GEN_INT (UNITS_PER_WORD))));

      emit_label (varargs_label);
      LABEL_NUSES (varargs_label) = 1;
    }
}

/* We may have to tell the dataflow pass that the split stack prologue
   is initializing a scratch register.  */

static void
ix86_live_on_entry (bitmap regs)
{
  if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
    {
      gcc_assert (flag_split_stack);
      bitmap_set_bit (regs, split_stack_prologue_scratch_regno ());
    }
}

/* Extract the parts of an RTL expression that is a valid memory address
   for an instruction.  Return 0 if the structure of the address is
   grossly off.  Return -1 if the address contains ASHIFT, so it is not
   strictly valid, but still used for computing length of lea instruction.  */

int
ix86_decompose_address (rtx addr, struct ix86_address *out)
{
  rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
  rtx base_reg, index_reg;
  HOST_WIDE_INT scale = 1;
  rtx scale_rtx = NULL_RTX;
  rtx tmp;
  int retval = 1;
  addr_space_t seg = ADDR_SPACE_GENERIC;

  /* Allow zero-extended SImode addresses,
     they will be emitted with addr32 prefix.  */
  if (TARGET_64BIT && GET_MODE (addr) == DImode)
    {
      if (GET_CODE (addr) == ZERO_EXTEND
	  && GET_MODE (XEXP (addr, 0)) == SImode)
	{
	  addr = XEXP (addr, 0);
	  if (CONST_INT_P (addr))
	    return 0;
	}	      
      else if (GET_CODE (addr) == AND
	       && const_32bit_mask (XEXP (addr, 1), DImode))
	{
	  addr = lowpart_subreg (SImode, XEXP (addr, 0), DImode);
	  if (addr == NULL_RTX)
	    return 0;

	  if (CONST_INT_P (addr))
	    return 0;
	}
    }

  /* Allow SImode subregs of DImode addresses,
     they will be emitted with addr32 prefix.  */
  if (TARGET_64BIT && GET_MODE (addr) == SImode)
    {
      if (SUBREG_P (addr)
	  && GET_MODE (SUBREG_REG (addr)) == DImode)
	{
	  addr = SUBREG_REG (addr);
	  if (CONST_INT_P (addr))
	    return 0;
	}
    }

  if (REG_P (addr))
    base = addr;
  else if (SUBREG_P (addr))
    {
      if (REG_P (SUBREG_REG (addr)))
	base = addr;
      else
	return 0;
    }
  else if (GET_CODE (addr) == PLUS)
    {
      rtx addends[4], op;
      int n = 0, i;

      op = addr;
      do
	{
	  if (n >= 4)
	    return 0;
	  addends[n++] = XEXP (op, 1);
	  op = XEXP (op, 0);
	}
      while (GET_CODE (op) == PLUS);
      if (n >= 4)
	return 0;
      addends[n] = op;

      for (i = n; i >= 0; --i)
	{
	  op = addends[i];
	  switch (GET_CODE (op))
	    {
	    case MULT:
	      if (index)
		return 0;
	      index = XEXP (op, 0);
	      scale_rtx = XEXP (op, 1);
	      break;

	    case ASHIFT:
	      if (index)
		return 0;
	      index = XEXP (op, 0);
	      tmp = XEXP (op, 1);
	      if (!CONST_INT_P (tmp))
		return 0;
	      scale = INTVAL (tmp);
	      if ((unsigned HOST_WIDE_INT) scale > 3)
		return 0;
	      scale = 1 << scale;
	      break;

	    case ZERO_EXTEND:
	      op = XEXP (op, 0);
	      if (GET_CODE (op) != UNSPEC)
		return 0;
	      /* FALLTHRU */

	    case UNSPEC:
	      if (XINT (op, 1) == UNSPEC_TP
	          && TARGET_TLS_DIRECT_SEG_REFS
	          && seg == ADDR_SPACE_GENERIC)
		seg = DEFAULT_TLS_SEG_REG;
	      else
		return 0;
	      break;

	    case SUBREG:
	      if (!REG_P (SUBREG_REG (op)))
		return 0;
	      /* FALLTHRU */

	    case REG:
	      if (!base)
		base = op;
	      else if (!index)
		index = op;
	      else
		return 0;
	      break;

	    case CONST:
	    case CONST_INT:
	    case SYMBOL_REF:
	    case LABEL_REF:
	      if (disp)
		return 0;
	      disp = op;
	      break;

	    default:
	      return 0;
	    }
	}
    }
  else if (GET_CODE (addr) == MULT)
    {
      index = XEXP (addr, 0);		/* index*scale */
      scale_rtx = XEXP (addr, 1);
    }
  else if (GET_CODE (addr) == ASHIFT)
    {
      /* We're called for lea too, which implements ashift on occasion.  */
      index = XEXP (addr, 0);
      tmp = XEXP (addr, 1);
      if (!CONST_INT_P (tmp))
	return 0;
      scale = INTVAL (tmp);
      if ((unsigned HOST_WIDE_INT) scale > 3)
	return 0;
      scale = 1 << scale;
      retval = -1;
    }
  else
    disp = addr;			/* displacement */

  if (index)
    {
      if (REG_P (index))
	;
      else if (SUBREG_P (index)
	       && REG_P (SUBREG_REG (index)))
	;
      else
	return 0;
    }

  /* Extract the integral value of scale.  */
  if (scale_rtx)
    {
      if (!CONST_INT_P (scale_rtx))
	return 0;
      scale = INTVAL (scale_rtx);
    }

  base_reg = base && SUBREG_P (base) ? SUBREG_REG (base) : base;
  index_reg = index && SUBREG_P (index) ? SUBREG_REG (index) : index;

  /* Avoid useless 0 displacement.  */
  if (disp == const0_rtx && (base || index))
    disp = NULL_RTX;

  /* Allow arg pointer and stack pointer as index if there is not scaling.  */
  if (base_reg && index_reg && scale == 1
      && (index_reg == arg_pointer_rtx
	  || index_reg == frame_pointer_rtx
	  || (REG_P (index_reg) && REGNO (index_reg) == STACK_POINTER_REGNUM)))
    {
      std::swap (base, index);
      std::swap (base_reg, index_reg);
    }

  /* Special case: %ebp cannot be encoded as a base without a displacement.
     Similarly %r13.  */
  if (!disp
      && base_reg
      && (base_reg == hard_frame_pointer_rtx
	  || base_reg == frame_pointer_rtx
	  || base_reg == arg_pointer_rtx
	  || (REG_P (base_reg)
	      && (REGNO (base_reg) == HARD_FRAME_POINTER_REGNUM
		  || REGNO (base_reg) == R13_REG))))
    disp = const0_rtx;

  /* Special case: on K6, [%esi] makes the instruction vector decoded.
     Avoid this by transforming to [%esi+0].
     Reload calls address legitimization without cfun defined, so we need
     to test cfun for being non-NULL. */
  if (TARGET_K6 && cfun && optimize_function_for_speed_p (cfun)
      && base_reg && !index_reg && !disp
      && REG_P (base_reg) && REGNO (base_reg) == SI_REG)
    disp = const0_rtx;

  /* Special case: encode reg+reg instead of reg*2.  */
  if (!base && index && scale == 2)
    base = index, base_reg = index_reg, scale = 1;

  /* Special case: scaling cannot be encoded without base or displacement.  */
  if (!base && !disp && index && scale != 1)
    disp = const0_rtx;

  out->base = base;
  out->index = index;
  out->disp = disp;
  out->scale = scale;
  out->seg = seg;

  return retval;
}

/* Return cost of the memory address x.
   For i386, it is better to use a complex address than let gcc copy
   the address into a reg and make a new pseudo.  But not if the address
   requires to two regs - that would mean more pseudos with longer
   lifetimes.  */
static int
ix86_address_cost (rtx x, machine_mode, addr_space_t, bool)
{
  struct ix86_address parts;
  int cost = 1;
  int ok = ix86_decompose_address (x, &parts);

  gcc_assert (ok);

  if (parts.base && SUBREG_P (parts.base))
    parts.base = SUBREG_REG (parts.base);
  if (parts.index && SUBREG_P (parts.index))
    parts.index = SUBREG_REG (parts.index);

  /* Attempt to minimize number of registers in the address by increasing
     address cost for each used register.  We don't increase address cost
     for "pic_offset_table_rtx".  When a memopt with "pic_offset_table_rtx"
     is not invariant itself it most likely means that base or index is not
     invariant.  Therefore only "pic_offset_table_rtx" could be hoisted out,
     which is not profitable for x86.  */
  if (parts.base
      && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER)
      && (current_pass->type == GIMPLE_PASS
	  || !pic_offset_table_rtx
	  || !REG_P (parts.base)
	  || REGNO (pic_offset_table_rtx) != REGNO (parts.base)))
    cost++;

  if (parts.index
      && (!REG_P (parts.index) || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)
      && (current_pass->type == GIMPLE_PASS
	  || !pic_offset_table_rtx
	  || !REG_P (parts.index)
	  || REGNO (pic_offset_table_rtx) != REGNO (parts.index)))
    cost++;

  /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
     since it's predecode logic can't detect the length of instructions
     and it degenerates to vector decoded.  Increase cost of such
     addresses here.  The penalty is minimally 2 cycles.  It may be worthwhile
     to split such addresses or even refuse such addresses at all.

     Following addressing modes are affected:
      [base+scale*index]
      [scale*index+disp]
      [base+index]

     The first and last case  may be avoidable by explicitly coding the zero in
     memory address, but I don't have AMD-K6 machine handy to check this
     theory.  */

  if (TARGET_K6
      && ((!parts.disp && parts.base && parts.index && parts.scale != 1)
	  || (parts.disp && !parts.base && parts.index && parts.scale != 1)
	  || (!parts.disp && parts.base && parts.index && parts.scale == 1)))
    cost += 10;

  return cost;
}

/* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
   this is used for to form addresses to local data when -fPIC is in
   use.  */

static bool
darwin_local_data_pic (rtx disp)
{
  return (GET_CODE (disp) == UNSPEC
	  && XINT (disp, 1) == UNSPEC_MACHOPIC_OFFSET);
}

/* True if operand X should be loaded from GOT.  */

bool
ix86_force_load_from_GOT_p (rtx x)
{
  return ((TARGET_64BIT || HAVE_AS_IX86_GOT32X)
	  && !TARGET_PECOFF && !TARGET_MACHO
	  && !flag_plt && !flag_pic
	  && ix86_cmodel != CM_LARGE
	  && GET_CODE (x) == SYMBOL_REF
	  && SYMBOL_REF_FUNCTION_P (x)
	  && !SYMBOL_REF_LOCAL_P (x));
}

/* Determine if a given RTX is a valid constant.  We already know this
   satisfies CONSTANT_P.  */

static bool
ix86_legitimate_constant_p (machine_mode mode, rtx x)
{
  /* Pointer bounds constants are not valid.  */
  if (POINTER_BOUNDS_MODE_P (GET_MODE (x)))
    return false;

  switch (GET_CODE (x))
    {
    case CONST:
      x = XEXP (x, 0);

      if (GET_CODE (x) == PLUS)
	{
	  if (!CONST_INT_P (XEXP (x, 1)))
	    return false;
	  x = XEXP (x, 0);
	}

      if (TARGET_MACHO && darwin_local_data_pic (x))
	return true;

      /* Only some unspecs are valid as "constants".  */
      if (GET_CODE (x) == UNSPEC)
	switch (XINT (x, 1))
	  {
	  case UNSPEC_GOT:
	  case UNSPEC_GOTOFF:
	  case UNSPEC_PLTOFF:
	    return TARGET_64BIT;
	  case UNSPEC_TPOFF:
	  case UNSPEC_NTPOFF:
	    x = XVECEXP (x, 0, 0);
	    return (GET_CODE (x) == SYMBOL_REF
		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
	  case UNSPEC_DTPOFF:
	    x = XVECEXP (x, 0, 0);
	    return (GET_CODE (x) == SYMBOL_REF
		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
	  default:
	    return false;
	  }

      /* We must have drilled down to a symbol.  */
      if (GET_CODE (x) == LABEL_REF)
	return true;
      if (GET_CODE (x) != SYMBOL_REF)
	return false;
      /* FALLTHRU */

    case SYMBOL_REF:
      /* TLS symbols are never valid.  */
      if (SYMBOL_REF_TLS_MODEL (x))
	return false;

      /* DLLIMPORT symbols are never valid.  */
      if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
	  && SYMBOL_REF_DLLIMPORT_P (x))
	return false;

#if TARGET_MACHO
      /* mdynamic-no-pic */
      if (MACHO_DYNAMIC_NO_PIC_P)
	return machopic_symbol_defined_p (x);
#endif

      /* External function address should be loaded
	 via the GOT slot to avoid PLT.  */
      if (ix86_force_load_from_GOT_p (x))
	return false;

      break;

    CASE_CONST_SCALAR_INT:
      switch (mode)
	{
	case TImode:
	  if (TARGET_64BIT)
	    return true;
	  /* FALLTHRU */
	case OImode:
	case XImode:
	  if (!standard_sse_constant_p (x, mode))
	    return false;
	default:
	  break;
	}
      break;

    case CONST_VECTOR:
      if (!standard_sse_constant_p (x, mode))
	return false;

    default:
      break;
    }

  /* Otherwise we handle everything else in the move patterns.  */
  return true;
}

/* Determine if it's legal to put X into the constant pool.  This
   is not possible for the address of thread-local symbols, which
   is checked above.  */

static bool
ix86_cannot_force_const_mem (machine_mode mode, rtx x)
{
  /* We can put any immediate constant in memory.  */
  switch (GET_CODE (x))
    {
    CASE_CONST_ANY:
      return false;

    default:
      break;
    }

  return !ix86_legitimate_constant_p (mode, x);
}

/*  Nonzero if the symbol is marked as dllimport, or as stub-variable,
    otherwise zero.  */

static bool
is_imported_p (rtx x)
{
  if (!TARGET_DLLIMPORT_DECL_ATTRIBUTES
      || GET_CODE (x) != SYMBOL_REF)
    return false;

  return SYMBOL_REF_DLLIMPORT_P (x) || SYMBOL_REF_STUBVAR_P (x);
}


/* Nonzero if the constant value X is a legitimate general operand
   when generating PIC code.  It is given that flag_pic is on and
   that X satisfies CONSTANT_P.  */

bool
legitimate_pic_operand_p (rtx x)
{
  rtx inner;

  switch (GET_CODE (x))
    {
    case CONST:
      inner = XEXP (x, 0);
      if (GET_CODE (inner) == PLUS
	  && CONST_INT_P (XEXP (inner, 1)))
	inner = XEXP (inner, 0);

      /* Only some unspecs are valid as "constants".  */
      if (GET_CODE (inner) == UNSPEC)
	switch (XINT (inner, 1))
	  {
	  case UNSPEC_GOT:
	  case UNSPEC_GOTOFF:
	  case UNSPEC_PLTOFF:
	    return TARGET_64BIT;
	  case UNSPEC_TPOFF:
	    x = XVECEXP (inner, 0, 0);
	    return (GET_CODE (x) == SYMBOL_REF
		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
	  case UNSPEC_MACHOPIC_OFFSET:
	    return legitimate_pic_address_disp_p (x);
	  default:
	    return false;
	  }
      /* FALLTHRU */

    case SYMBOL_REF:
    case LABEL_REF:
      return legitimate_pic_address_disp_p (x);

    default:
      return true;
    }
}

/* Determine if a given CONST RTX is a valid memory displacement
   in PIC mode.  */

bool
legitimate_pic_address_disp_p (rtx disp)
{
  bool saw_plus;

  /* In 64bit mode we can allow direct addresses of symbols and labels
     when they are not dynamic symbols.  */
  if (TARGET_64BIT)
    {
      rtx op0 = disp, op1;

      switch (GET_CODE (disp))
	{
	case LABEL_REF:
	  return true;

	case CONST:
	  if (GET_CODE (XEXP (disp, 0)) != PLUS)
	    break;
	  op0 = XEXP (XEXP (disp, 0), 0);
	  op1 = XEXP (XEXP (disp, 0), 1);
	  if (!CONST_INT_P (op1)
	      || INTVAL (op1) >= 16*1024*1024
	      || INTVAL (op1) < -16*1024*1024)
            break;
	  if (GET_CODE (op0) == LABEL_REF)
	    return true;
	  if (GET_CODE (op0) == CONST
	      && GET_CODE (XEXP (op0, 0)) == UNSPEC
	      && XINT (XEXP (op0, 0), 1) == UNSPEC_PCREL)
	    return true;
	  if (GET_CODE (op0) == UNSPEC
	      && XINT (op0, 1) == UNSPEC_PCREL)
	    return true;
	  if (GET_CODE (op0) != SYMBOL_REF)
	    break;
	  /* FALLTHRU */

	case SYMBOL_REF:
	  /* TLS references should always be enclosed in UNSPEC.
	     The dllimported symbol needs always to be resolved.  */
	  if (SYMBOL_REF_TLS_MODEL (op0)
	      || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && SYMBOL_REF_DLLIMPORT_P (op0)))
	    return false;

	  if (TARGET_PECOFF)
	    {
	      if (is_imported_p (op0))
		return true;

	      if (SYMBOL_REF_FAR_ADDR_P (op0)
		  || !SYMBOL_REF_LOCAL_P (op0))
		break;

	      /* Function-symbols need to be resolved only for
	         large-model.
	         For the small-model we don't need to resolve anything
	         here.  */
	      if ((ix86_cmodel != CM_LARGE_PIC
	           && SYMBOL_REF_FUNCTION_P (op0))
		  || ix86_cmodel == CM_SMALL_PIC)
		return true;
	      /* Non-external symbols don't need to be resolved for
	         large, and medium-model.  */
	      if ((ix86_cmodel == CM_LARGE_PIC
		   || ix86_cmodel == CM_MEDIUM_PIC)
		  && !SYMBOL_REF_EXTERNAL_P (op0))
		return true;
	    }
	  else if (!SYMBOL_REF_FAR_ADDR_P (op0)
		   && (SYMBOL_REF_LOCAL_P (op0)
		       || (HAVE_LD_PIE_COPYRELOC
			   && flag_pie
			   && !SYMBOL_REF_WEAK (op0)
			   && !SYMBOL_REF_FUNCTION_P (op0)))
		   && ix86_cmodel != CM_LARGE_PIC)
	    return true;
	  break;

	default:
	  break;
	}
    }
  if (GET_CODE (disp) != CONST)
    return false;
  disp = XEXP (disp, 0);

  if (TARGET_64BIT)
    {
      /* We are unsafe to allow PLUS expressions.  This limit allowed distance
         of GOT tables.  We should not need these anyway.  */
      if (GET_CODE (disp) != UNSPEC
	  || (XINT (disp, 1) != UNSPEC_GOTPCREL
	      && XINT (disp, 1) != UNSPEC_GOTOFF
	      && XINT (disp, 1) != UNSPEC_PCREL
	      && XINT (disp, 1) != UNSPEC_PLTOFF))
	return false;

      if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF
	  && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF)
	return false;
      return true;
    }

  saw_plus = false;
  if (GET_CODE (disp) == PLUS)
    {
      if (!CONST_INT_P (XEXP (disp, 1)))
	return false;
      disp = XEXP (disp, 0);
      saw_plus = true;
    }

  if (TARGET_MACHO && darwin_local_data_pic (disp))
    return true;

  if (GET_CODE (disp) != UNSPEC)
    return false;

  switch (XINT (disp, 1))
    {
    case UNSPEC_GOT:
      if (saw_plus)
	return false;
      /* We need to check for both symbols and labels because VxWorks loads
	 text labels with @GOT rather than @GOTOFF.  See gotoff_operand for
	 details.  */
      return (GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
	      || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF);
    case UNSPEC_GOTOFF:
      /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
	 While ABI specify also 32bit relocation but we don't produce it in
	 small PIC model at all.  */
      if ((GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
	   || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF)
	  && !TARGET_64BIT)
        return !TARGET_PECOFF && gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
      return false;
    case UNSPEC_GOTTPOFF:
    case UNSPEC_GOTNTPOFF:
    case UNSPEC_INDNTPOFF:
      if (saw_plus)
	return false;
      disp = XVECEXP (disp, 0, 0);
      return (GET_CODE (disp) == SYMBOL_REF
	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
    case UNSPEC_NTPOFF:
      disp = XVECEXP (disp, 0, 0);
      return (GET_CODE (disp) == SYMBOL_REF
	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
    case UNSPEC_DTPOFF:
      disp = XVECEXP (disp, 0, 0);
      return (GET_CODE (disp) == SYMBOL_REF
	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
    }

  return false;
}

/* Determine if op is suitable RTX for an address register.
   Return naked register if a register or a register subreg is
   found, otherwise return NULL_RTX.  */

static rtx
ix86_validate_address_register (rtx op)
{
  machine_mode mode = GET_MODE (op);

  /* Only SImode or DImode registers can form the address.  */
  if (mode != SImode && mode != DImode)
    return NULL_RTX;

  if (REG_P (op))
    return op;
  else if (SUBREG_P (op))
    {
      rtx reg = SUBREG_REG (op);

      if (!REG_P (reg))
	return NULL_RTX;

      mode = GET_MODE (reg);

      /* Don't allow SUBREGs that span more than a word.  It can
	 lead to spill failures when the register is one word out
	 of a two word structure.  */
      if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
	return NULL_RTX;

      /* Allow only SUBREGs of non-eliminable hard registers.  */
      if (register_no_elim_operand (reg, mode))
	return reg;
    }

  /* Op is not a register.  */
  return NULL_RTX;
}

/* Recognizes RTL expressions that are valid memory addresses for an
   instruction.  The MODE argument is the machine mode for the MEM
   expression that wants to use this address.

   It only recognizes address in canonical form.  LEGITIMIZE_ADDRESS should
   convert common non-canonical forms to canonical form so that they will
   be recognized.  */

static bool
ix86_legitimate_address_p (machine_mode, rtx addr, bool strict)
{
  struct ix86_address parts;
  rtx base, index, disp;
  HOST_WIDE_INT scale;
  addr_space_t seg;

  if (ix86_decompose_address (addr, &parts) <= 0)
    /* Decomposition failed.  */
    return false;

  base = parts.base;
  index = parts.index;
  disp = parts.disp;
  scale = parts.scale;
  seg = parts.seg;

  /* Validate base register.  */
  if (base)
    {
      rtx reg = ix86_validate_address_register (base);

      if (reg == NULL_RTX)
	return false;

      if ((strict && ! REG_OK_FOR_BASE_STRICT_P (reg))
	  || (! strict && ! REG_OK_FOR_BASE_NONSTRICT_P (reg)))
	/* Base is not valid.  */
	return false;
    }

  /* Validate index register.  */
  if (index)
    {
      rtx reg = ix86_validate_address_register (index);

      if (reg == NULL_RTX)
	return false;

      if ((strict && ! REG_OK_FOR_INDEX_STRICT_P (reg))
	  || (! strict && ! REG_OK_FOR_INDEX_NONSTRICT_P (reg)))
	/* Index is not valid.  */
	return false;
    }

  /* Index and base should have the same mode.  */
  if (base && index
      && GET_MODE (base) != GET_MODE (index))
    return false;

  /* Address override works only on the (%reg) part of %fs:(%reg).  */
  if (seg != ADDR_SPACE_GENERIC
      && ((base && GET_MODE (base) != word_mode)
	  || (index && GET_MODE (index) != word_mode)))
    return false;

  /* Validate scale factor.  */
  if (scale != 1)
    {
      if (!index)
	/* Scale without index.  */
	return false;

      if (scale != 2 && scale != 4 && scale != 8)
	/* Scale is not a valid multiplier.  */
	return false;
    }

  /* Validate displacement.  */
  if (disp)
    {
      if (GET_CODE (disp) == CONST
	  && GET_CODE (XEXP (disp, 0)) == UNSPEC
	  && XINT (XEXP (disp, 0), 1) != UNSPEC_MACHOPIC_OFFSET)
	switch (XINT (XEXP (disp, 0), 1))
	  {
	  /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit
	     when used.  While ABI specify also 32bit relocations, we
	     don't produce them at all and use IP relative instead.
	     Allow GOT in 32bit mode for both PIC and non-PIC if symbol
	     should be loaded via GOT.  */
	  case UNSPEC_GOT:
	    if (!TARGET_64BIT
		&& ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
	      goto is_legitimate_pic;
	    /* FALLTHRU */
	  case UNSPEC_GOTOFF:
	    gcc_assert (flag_pic);
	    if (!TARGET_64BIT)
	      goto is_legitimate_pic;

	    /* 64bit address unspec.  */
	    return false;

	  case UNSPEC_GOTPCREL:
	    if (ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
	      goto is_legitimate_pic;
	    /* FALLTHRU */
	  case UNSPEC_PCREL:
	    gcc_assert (flag_pic);
	    goto is_legitimate_pic;

	  case UNSPEC_GOTTPOFF:
	  case UNSPEC_GOTNTPOFF:
	  case UNSPEC_INDNTPOFF:
	  case UNSPEC_NTPOFF:
	  case UNSPEC_DTPOFF:
	    break;

	  case UNSPEC_STACK_CHECK:
	    gcc_assert (flag_split_stack);
	    break;

	  default:
	    /* Invalid address unspec.  */
	    return false;
	  }

      else if (SYMBOLIC_CONST (disp)
	       && (flag_pic
		   || (TARGET_MACHO
#if TARGET_MACHO
		       && MACHOPIC_INDIRECT
		       && !machopic_operand_p (disp)
#endif
	       )))
	{

	is_legitimate_pic:
	  if (TARGET_64BIT && (index || base))
	    {
	      /* foo@dtpoff(%rX) is ok.  */
	      if (GET_CODE (disp) != CONST
		  || GET_CODE (XEXP (disp, 0)) != PLUS
		  || GET_CODE (XEXP (XEXP (disp, 0), 0)) != UNSPEC
		  || !CONST_INT_P (XEXP (XEXP (disp, 0), 1))
		  || (XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_DTPOFF
		      && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_NTPOFF))
		/* Non-constant pic memory reference.  */
		return false;
	    }
	  else if ((!TARGET_MACHO || flag_pic)
		    && ! legitimate_pic_address_disp_p (disp))
	    /* Displacement is an invalid pic construct.  */
	    return false;
#if TARGET_MACHO
	  else if (MACHO_DYNAMIC_NO_PIC_P
		   && !ix86_legitimate_constant_p (Pmode, disp))
	    /* displacment must be referenced via non_lazy_pointer */
	    return false;
#endif

          /* This code used to verify that a symbolic pic displacement
	     includes the pic_offset_table_rtx register.

	     While this is good idea, unfortunately these constructs may
	     be created by "adds using lea" optimization for incorrect
	     code like:

	     int a;
	     int foo(int i)
	       {
	         return *(&a+i);
	       }

	     This code is nonsensical, but results in addressing
	     GOT table with pic_offset_table_rtx base.  We can't
	     just refuse it easily, since it gets matched by
	     "addsi3" pattern, that later gets split to lea in the
	     case output register differs from input.  While this
	     can be handled by separate addsi pattern for this case
	     that never results in lea, this seems to be easier and
	     correct fix for crash to disable this test.  */
	}
      else if (GET_CODE (disp) != LABEL_REF
	       && !CONST_INT_P (disp)
	       && (GET_CODE (disp) != CONST
		   || !ix86_legitimate_constant_p (Pmode, disp))
	       && (GET_CODE (disp) != SYMBOL_REF
		   || !ix86_legitimate_constant_p (Pmode, disp)))
	/* Displacement is not constant.  */
	return false;
      else if (TARGET_64BIT
	       && !x86_64_immediate_operand (disp, VOIDmode))
	/* Displacement is out of range.  */
	return false;
      /* In x32 mode, constant addresses are sign extended to 64bit, so
	 we have to prevent addresses from 0x80000000 to 0xffffffff.  */
      else if (TARGET_X32 && !(index || base)
	       && CONST_INT_P (disp)
	       && val_signbit_known_set_p (SImode, INTVAL (disp)))
	return false;
    }

  /* Everything looks valid.  */
  return true;
}

/* Determine if a given RTX is a valid constant address.  */

bool
constant_address_p (rtx x)
{
  return CONSTANT_P (x) && ix86_legitimate_address_p (Pmode, x, 1);
}

/* Return a unique alias set for the GOT.  */

static alias_set_type
ix86_GOT_alias_set (void)
{
  static alias_set_type set = -1;
  if (set == -1)
    set = new_alias_set ();
  return set;
}

/* Return a legitimate reference for ORIG (an address) using the
   register REG.  If REG is 0, a new pseudo is generated.

   There are two types of references that must be handled:

   1. Global data references must load the address from the GOT, via
      the PIC reg.  An insn is emitted to do this load, and the reg is
      returned.

   2. Static data references, constant pool addresses, and code labels
      compute the address as an offset from the GOT, whose base is in
      the PIC reg.  Static data objects have SYMBOL_FLAG_LOCAL set to
      differentiate them from global data objects.  The returned
      address is the PIC reg + an unspec constant.

   TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
   reg also appears in the address.  */

static rtx
legitimize_pic_address (rtx orig, rtx reg)
{
  rtx addr = orig;
  rtx new_rtx = orig;

#if TARGET_MACHO
  if (TARGET_MACHO && !TARGET_64BIT)
    {
      if (reg == 0)
	reg = gen_reg_rtx (Pmode);
      /* Use the generic Mach-O PIC machinery.  */
      return machopic_legitimize_pic_address (orig, GET_MODE (orig), reg);
    }
#endif

  if (TARGET_64BIT && TARGET_DLLIMPORT_DECL_ATTRIBUTES)
    {
      rtx tmp = legitimize_pe_coff_symbol (addr, true);
      if (tmp)
        return tmp;
    }

  if (TARGET_64BIT && legitimate_pic_address_disp_p (addr))
    new_rtx = addr;
  else if ((!TARGET_64BIT
	    || /* TARGET_64BIT && */ ix86_cmodel != CM_SMALL_PIC)
	   && !TARGET_PECOFF
	   && gotoff_operand (addr, Pmode))
    {
      /* This symbol may be referenced via a displacement
	 from the PIC base address (@GOTOFF).  */
      if (GET_CODE (addr) == CONST)
	addr = XEXP (addr, 0);

      if (GET_CODE (addr) == PLUS)
	  {
            new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
				      UNSPEC_GOTOFF);
	    new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
	  }
	else
          new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);

      new_rtx = gen_rtx_CONST (Pmode, new_rtx);

      if (TARGET_64BIT)
	new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);

      if (reg != 0)
	{
 	  gcc_assert (REG_P (reg));
	  new_rtx = expand_simple_binop (Pmode, PLUS, pic_offset_table_rtx,
					 new_rtx, reg, 1, OPTAB_DIRECT);
 	}
      else
	new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
    }
  else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
	   /* We can't use @GOTOFF for text labels
	      on VxWorks, see gotoff_operand.  */
	   || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
    {
      rtx tmp = legitimize_pe_coff_symbol (addr, true);
      if (tmp)
        return tmp;

      /* For x64 PE-COFF there is no GOT table,
	 so we use address directly.  */
      if (TARGET_64BIT && TARGET_PECOFF)
	{
	  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_PCREL);
	  new_rtx = gen_rtx_CONST (Pmode, new_rtx);
	}
      else if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC)
	{
	  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
				    UNSPEC_GOTPCREL);
	  new_rtx = gen_rtx_CONST (Pmode, new_rtx);
	  new_rtx = gen_const_mem (Pmode, new_rtx);
	  set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
	}
      else
	{
	  /* This symbol must be referenced via a load
	     from the Global Offset Table (@GOT).  */
	  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
	  new_rtx = gen_rtx_CONST (Pmode, new_rtx);
	  if (TARGET_64BIT)
	    new_rtx = force_reg (Pmode, new_rtx);
	  new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
	  new_rtx = gen_const_mem (Pmode, new_rtx);
	  set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
	}

      new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
    }
  else
    {
      if (CONST_INT_P (addr)
	  && !x86_64_immediate_operand (addr, VOIDmode))
	new_rtx = copy_to_suggested_reg (addr, reg, Pmode);
      else if (GET_CODE (addr) == CONST)
	{
	  addr = XEXP (addr, 0);

	  /* We must match stuff we generate before.  Assume the only
	     unspecs that can get here are ours.  Not that we could do
	     anything with them anyway....  */
	  if (GET_CODE (addr) == UNSPEC
	      || (GET_CODE (addr) == PLUS
		  && GET_CODE (XEXP (addr, 0)) == UNSPEC))
	    return orig;
	  gcc_assert (GET_CODE (addr) == PLUS);
	}

      if (GET_CODE (addr) == PLUS)
	{
	  rtx op0 = XEXP (addr, 0), op1 = XEXP (addr, 1);

	  /* Check first to see if this is a constant
	     offset from a @GOTOFF symbol reference.  */
	  if (!TARGET_PECOFF
	      && gotoff_operand (op0, Pmode)
	      && CONST_INT_P (op1))
	    {
	      if (!TARGET_64BIT)
		{
		  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op0),
					    UNSPEC_GOTOFF);
		  new_rtx = gen_rtx_PLUS (Pmode, new_rtx, op1);
		  new_rtx = gen_rtx_CONST (Pmode, new_rtx);

		  if (reg != 0)
		    {
		      gcc_assert (REG_P (reg));
		      new_rtx = expand_simple_binop (Pmode, PLUS,
						     pic_offset_table_rtx,
						     new_rtx, reg, 1,
						     OPTAB_DIRECT);
		    }
		  else
		    new_rtx
		      = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
		}
	      else
		{
		  if (INTVAL (op1) < -16*1024*1024
		      || INTVAL (op1) >= 16*1024*1024)
		    {
		      if (!x86_64_immediate_operand (op1, Pmode))
			op1 = force_reg (Pmode, op1);

		      new_rtx
			= gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), op1);
		    }
		}
	    }
	  else
	    {
	      rtx base = legitimize_pic_address (op0, reg);
	      machine_mode mode = GET_MODE (base);
	      new_rtx
	        = legitimize_pic_address (op1, base == reg ? NULL_RTX : reg);

	      if (CONST_INT_P (new_rtx))
		{
		  if (INTVAL (new_rtx) < -16*1024*1024
		      || INTVAL (new_rtx) >= 16*1024*1024)
		    {
		      if (!x86_64_immediate_operand (new_rtx, mode))
			new_rtx = force_reg (mode, new_rtx);

		      new_rtx
		        = gen_rtx_PLUS (mode, force_reg (mode, base), new_rtx);
		    }
		  else
		    new_rtx = plus_constant (mode, base, INTVAL (new_rtx));
		}
	      else
		{
		  /* For %rip addressing, we have to use
		     just disp32, not base nor index.  */
		  if (TARGET_64BIT
		      && (GET_CODE (base) == SYMBOL_REF
			  || GET_CODE (base) == LABEL_REF))
		    base = force_reg (mode, base);
		  if (GET_CODE (new_rtx) == PLUS
		      && CONSTANT_P (XEXP (new_rtx, 1)))
		    {
		      base = gen_rtx_PLUS (mode, base, XEXP (new_rtx, 0));
		      new_rtx = XEXP (new_rtx, 1);
		    }
		  new_rtx = gen_rtx_PLUS (mode, base, new_rtx);
		}
	    }
	}
    }
  return new_rtx;
}

/* Load the thread pointer.  If TO_REG is true, force it into a register.  */

static rtx
get_thread_pointer (machine_mode tp_mode, bool to_reg)
{
  rtx tp = gen_rtx_UNSPEC (ptr_mode, gen_rtvec (1, const0_rtx), UNSPEC_TP);

  if (GET_MODE (tp) != tp_mode)
    {
      gcc_assert (GET_MODE (tp) == SImode);
      gcc_assert (tp_mode == DImode);

      tp = gen_rtx_ZERO_EXTEND (tp_mode, tp);
    }

  if (to_reg)
    tp = copy_to_mode_reg (tp_mode, tp);

  return tp;
}

/* Construct the SYMBOL_REF for the tls_get_addr function.  */

static GTY(()) rtx ix86_tls_symbol;

static rtx
ix86_tls_get_addr (void)
{
  if (!ix86_tls_symbol)
    {
      const char *sym
	= ((TARGET_ANY_GNU_TLS && !TARGET_64BIT)
	   ? "___tls_get_addr" : "__tls_get_addr");

      ix86_tls_symbol = gen_rtx_SYMBOL_REF (Pmode, sym);
    }

  if (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF)
    {
      rtx unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, ix86_tls_symbol),
				   UNSPEC_PLTOFF);
      return gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
			   gen_rtx_CONST (Pmode, unspec));
    }

  return ix86_tls_symbol;
}

/* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol.  */

static GTY(()) rtx ix86_tls_module_base_symbol;

rtx
ix86_tls_module_base (void)
{
  if (!ix86_tls_module_base_symbol)
    {
      ix86_tls_module_base_symbol
	= gen_rtx_SYMBOL_REF (Pmode, "_TLS_MODULE_BASE_");

      SYMBOL_REF_FLAGS (ix86_tls_module_base_symbol)
	|= TLS_MODEL_GLOBAL_DYNAMIC << SYMBOL_FLAG_TLS_SHIFT;
    }

  return ix86_tls_module_base_symbol;
}

/* A subroutine of ix86_legitimize_address and ix86_expand_move.  FOR_MOV is
   false if we expect this to be used for a memory address and true if
   we expect to load the address into a register.  */

static rtx
legitimize_tls_address (rtx x, enum tls_model model, bool for_mov)
{
  rtx dest, base, off;
  rtx pic = NULL_RTX, tp = NULL_RTX;
  machine_mode tp_mode = Pmode;
  int type;

  /* Fall back to global dynamic model if tool chain cannot support local
     dynamic.  */
  if (TARGET_SUN_TLS && !TARGET_64BIT
      && !HAVE_AS_IX86_TLSLDMPLT && !HAVE_AS_IX86_TLSLDM
      && model == TLS_MODEL_LOCAL_DYNAMIC)
    model = TLS_MODEL_GLOBAL_DYNAMIC;

  switch (model)
    {
    case TLS_MODEL_GLOBAL_DYNAMIC:
      dest = gen_reg_rtx (Pmode);

      if (!TARGET_64BIT)
	{
	  if (flag_pic && !TARGET_PECOFF)
	    pic = pic_offset_table_rtx;
	  else
	    {
	      pic = gen_reg_rtx (Pmode);
	      emit_insn (gen_set_got (pic));
	    }
	}

      if (TARGET_GNU2_TLS)
	{
	  if (TARGET_64BIT)
	    emit_insn (gen_tls_dynamic_gnu2_64 (dest, x));
	  else
	    emit_insn (gen_tls_dynamic_gnu2_32 (dest, x, pic));

	  tp = get_thread_pointer (Pmode, true);
	  dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, tp, dest));

	  if (GET_MODE (x) != Pmode)
	    x = gen_rtx_ZERO_EXTEND (Pmode, x);

	  set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
	}
      else
	{
	  rtx caddr = ix86_tls_get_addr ();

	  if (TARGET_64BIT)
	    {
	      rtx rax = gen_rtx_REG (Pmode, AX_REG);
	      rtx_insn *insns;

	      start_sequence ();
	      emit_call_insn
		(ix86_gen_tls_global_dynamic_64 (rax, x, caddr));
	      insns = get_insns ();
	      end_sequence ();

	      if (GET_MODE (x) != Pmode)
		x = gen_rtx_ZERO_EXTEND (Pmode, x);

	      RTL_CONST_CALL_P (insns) = 1;
	      emit_libcall_block (insns, dest, rax, x);
	    }
	  else
	    emit_insn (gen_tls_global_dynamic_32 (dest, x, pic, caddr));
	}
      break;

    case TLS_MODEL_LOCAL_DYNAMIC:
      base = gen_reg_rtx (Pmode);

      if (!TARGET_64BIT)
	{
	  if (flag_pic)
	    pic = pic_offset_table_rtx;
	  else
	    {
	      pic = gen_reg_rtx (Pmode);
	      emit_insn (gen_set_got (pic));
	    }
	}

      if (TARGET_GNU2_TLS)
	{
	  rtx tmp = ix86_tls_module_base ();

	  if (TARGET_64BIT)
	    emit_insn (gen_tls_dynamic_gnu2_64 (base, tmp));
	  else
	    emit_insn (gen_tls_dynamic_gnu2_32 (base, tmp, pic));

	  tp = get_thread_pointer (Pmode, true);
	  set_unique_reg_note (get_last_insn (), REG_EQUAL,
			       gen_rtx_MINUS (Pmode, tmp, tp));
	}
      else
	{
	  rtx caddr = ix86_tls_get_addr ();

	  if (TARGET_64BIT)
	    {
	      rtx rax = gen_rtx_REG (Pmode, AX_REG);
	      rtx_insn *insns;
	      rtx eqv;

	      start_sequence ();
	      emit_call_insn
		(ix86_gen_tls_local_dynamic_base_64 (rax, caddr));
	      insns = get_insns ();
	      end_sequence ();

	      /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
		 share the LD_BASE result with other LD model accesses.  */
	      eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
				    UNSPEC_TLS_LD_BASE);

	      RTL_CONST_CALL_P (insns) = 1;
	      emit_libcall_block (insns, base, rax, eqv);
	    }
	  else
	    emit_insn (gen_tls_local_dynamic_base_32 (base, pic, caddr));
	}

      off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), UNSPEC_DTPOFF);
      off = gen_rtx_CONST (Pmode, off);

      dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, base, off));

      if (TARGET_GNU2_TLS)
	{
	  dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, dest, tp));

	  if (GET_MODE (x) != Pmode)
	    x = gen_rtx_ZERO_EXTEND (Pmode, x);

	  set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
	}
      break;

    case TLS_MODEL_INITIAL_EXEC:
      if (TARGET_64BIT)
	{
	  if (TARGET_SUN_TLS && !TARGET_X32)
	    {
	      /* The Sun linker took the AMD64 TLS spec literally
		 and can only handle %rax as destination of the
		 initial executable code sequence.  */

	      dest = gen_reg_rtx (DImode);
	      emit_insn (gen_tls_initial_exec_64_sun (dest, x));
	      return dest;
	    }

	  /* Generate DImode references to avoid %fs:(%reg32)
	     problems and linker IE->LE relaxation bug.  */
	  tp_mode = DImode;
	  pic = NULL;
	  type = UNSPEC_GOTNTPOFF;
	}
      else if (flag_pic)
	{
	  pic = pic_offset_table_rtx;
	  type = TARGET_ANY_GNU_TLS ? UNSPEC_GOTNTPOFF : UNSPEC_GOTTPOFF;
	}
      else if (!TARGET_ANY_GNU_TLS)
	{
	  pic = gen_reg_rtx (Pmode);
	  emit_insn (gen_set_got (pic));
	  type = UNSPEC_GOTTPOFF;
	}
      else
	{
	  pic = NULL;
	  type = UNSPEC_INDNTPOFF;
	}

      off = gen_rtx_UNSPEC (tp_mode, gen_rtvec (1, x), type);
      off = gen_rtx_CONST (tp_mode, off);
      if (pic)
	off = gen_rtx_PLUS (tp_mode, pic, off);
      off = gen_const_mem (tp_mode, off);
      set_mem_alias_set (off, ix86_GOT_alias_set ());

      if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
	{
	  base = get_thread_pointer (tp_mode,
				     for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
	  off = force_reg (tp_mode, off);
	  dest = gen_rtx_PLUS (tp_mode, base, off);
	  if (tp_mode != Pmode)
	    dest = convert_to_mode (Pmode, dest, 1);
	}
      else
	{
	  base = get_thread_pointer (Pmode, true);
	  dest = gen_reg_rtx (Pmode);
	  emit_insn (ix86_gen_sub3 (dest, base, off));
	}
      break;

    case TLS_MODEL_LOCAL_EXEC:
      off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
			    (TARGET_64BIT || TARGET_ANY_GNU_TLS)
			    ? UNSPEC_NTPOFF : UNSPEC_TPOFF);
      off = gen_rtx_CONST (Pmode, off);

      if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
	{
	  base = get_thread_pointer (Pmode,
				     for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
	  return gen_rtx_PLUS (Pmode, base, off);
	}
      else
	{
	  base = get_thread_pointer (Pmode, true);
	  dest = gen_reg_rtx (Pmode);
	  emit_insn (ix86_gen_sub3 (dest, base, off));
	}
      break;

    default:
      gcc_unreachable ();
    }

  return dest;
}

/* Create or return the unique __imp_DECL dllimport symbol corresponding
   to symbol DECL if BEIMPORT is true.  Otherwise create or return the
   unique refptr-DECL symbol corresponding to symbol DECL.  */

struct dllimport_hasher : ggc_cache_ptr_hash<tree_map>
{
  static inline hashval_t hash (tree_map *m) { return m->hash; }
  static inline bool
  equal (tree_map *a, tree_map *b)
  {
    return a->base.from == b->base.from;
  }

  static int
  keep_cache_entry (tree_map *&m)
  {
    return ggc_marked_p (m->base.from);
  }
};

static GTY((cache)) hash_table<dllimport_hasher> *dllimport_map;

static tree
get_dllimport_decl (tree decl, bool beimport)
{
  struct tree_map *h, in;
  const char *name;
  const char *prefix;
  size_t namelen, prefixlen;
  char *imp_name;
  tree to;
  rtx rtl;

  if (!dllimport_map)
    dllimport_map = hash_table<dllimport_hasher>::create_ggc (512);

  in.hash = htab_hash_pointer (decl);
  in.base.from = decl;
  tree_map **loc = dllimport_map->find_slot_with_hash (&in, in.hash, INSERT);
  h = *loc;
  if (h)
    return h->to;

  *loc = h = ggc_alloc<tree_map> ();
  h->hash = in.hash;
  h->base.from = decl;
  h->to = to = build_decl (DECL_SOURCE_LOCATION (decl),
			   VAR_DECL, NULL, ptr_type_node);
  DECL_ARTIFICIAL (to) = 1;
  DECL_IGNORED_P (to) = 1;
  DECL_EXTERNAL (to) = 1;
  TREE_READONLY (to) = 1;

  name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
  name = targetm.strip_name_encoding (name);
  if (beimport)
    prefix = name[0] == FASTCALL_PREFIX || user_label_prefix[0] == 0
      ? "*__imp_" : "*__imp__";
  else
    prefix = user_label_prefix[0] == 0 ? "*.refptr." : "*refptr.";
  namelen = strlen (name);
  prefixlen = strlen (prefix);
  imp_name = (char *) alloca (namelen + prefixlen + 1);
  memcpy (imp_name, prefix, prefixlen);
  memcpy (imp_name + prefixlen, name, namelen + 1);

  name = ggc_alloc_string (imp_name, namelen + prefixlen);
  rtl = gen_rtx_SYMBOL_REF (Pmode, name);
  SET_SYMBOL_REF_DECL (rtl, to);
  SYMBOL_REF_FLAGS (rtl) = SYMBOL_FLAG_LOCAL | SYMBOL_FLAG_STUBVAR;
  if (!beimport)
    {
      SYMBOL_REF_FLAGS (rtl) |= SYMBOL_FLAG_EXTERNAL;
#ifdef SUB_TARGET_RECORD_STUB
      SUB_TARGET_RECORD_STUB (name);
#endif
    }      

  rtl = gen_const_mem (Pmode, rtl);
  set_mem_alias_set (rtl, ix86_GOT_alias_set ());

  SET_DECL_RTL (to, rtl);
  SET_DECL_ASSEMBLER_NAME (to, get_identifier (name));

  return to;
}

/* Expand SYMBOL into its corresponding far-addresse symbol.
   WANT_REG is true if we require the result be a register.  */

static rtx
legitimize_pe_coff_extern_decl (rtx symbol, bool want_reg)
{
  tree imp_decl;
  rtx x;

  gcc_assert (SYMBOL_REF_DECL (symbol));
  imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol), false);

  x = DECL_RTL (imp_decl);
  if (want_reg)
    x = force_reg (Pmode, x);
  return x;
}

/* Expand SYMBOL into its corresponding dllimport symbol.  WANT_REG is
   true if we require the result be a register.  */

static rtx
legitimize_dllimport_symbol (rtx symbol, bool want_reg)
{
  tree imp_decl;
  rtx x;

  gcc_assert (SYMBOL_REF_DECL (symbol));
  imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol), true);

  x = DECL_RTL (imp_decl);
  if (want_reg)
    x = force_reg (Pmode, x);
  return x;
}

/* Expand SYMBOL into its corresponding dllimport or refptr symbol.  WANT_REG 
   is true if we require the result be a register.  */

static rtx
legitimize_pe_coff_symbol (rtx addr, bool inreg)
{
  if (!TARGET_PECOFF)
    return NULL_RTX;

  if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
    {
      if (GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_DLLIMPORT_P (addr))
	return legitimize_dllimport_symbol (addr, inreg);
      if (GET_CODE (addr) == CONST
	  && GET_CODE (XEXP (addr, 0)) == PLUS
	  && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
	  && SYMBOL_REF_DLLIMPORT_P (XEXP (XEXP (addr, 0), 0)))
	{
	  rtx t = legitimize_dllimport_symbol (XEXP (XEXP (addr, 0), 0), inreg);
	  return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
	}
    }

  if (ix86_cmodel != CM_LARGE_PIC && ix86_cmodel != CM_MEDIUM_PIC)
    return NULL_RTX;
  if (GET_CODE (addr) == SYMBOL_REF
      && !is_imported_p (addr)
      && SYMBOL_REF_EXTERNAL_P (addr)
      && SYMBOL_REF_DECL (addr))
    return legitimize_pe_coff_extern_decl (addr, inreg);

  if (GET_CODE (addr) == CONST
      && GET_CODE (XEXP (addr, 0)) == PLUS
      && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
      && !is_imported_p (XEXP (XEXP (addr, 0), 0))
      && SYMBOL_REF_EXTERNAL_P (XEXP (XEXP (addr, 0), 0))
      && SYMBOL_REF_DECL (XEXP (XEXP (addr, 0), 0)))
    {
      rtx t = legitimize_pe_coff_extern_decl (XEXP (XEXP (addr, 0), 0), inreg);
      return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
    }
  return NULL_RTX;
}

/* Try machine-dependent ways of modifying an illegitimate address
   to be legitimate.  If we find one, return the new, valid address.
   This macro is used in only one place: `memory_address' in explow.c.

   OLDX is the address as it was before break_out_memory_refs was called.
   In some cases it is useful to look at this to decide what needs to be done.

   It is always safe for this macro to do nothing.  It exists to recognize
   opportunities to optimize the output.

   For the 80386, we handle X+REG by loading X into a register R and
   using R+REG.  R will go in a general reg and indexing will be used.
   However, if REG is a broken-out memory address or multiplication,
   nothing needs to be done because REG can certainly go in a general reg.

   When -fpic is used, special handling is needed for symbolic references.
   See comments by legitimize_pic_address in i386.c for details.  */

static rtx
ix86_legitimize_address (rtx x, rtx, machine_mode mode)
{
  bool changed = false;
  unsigned log;

  log = GET_CODE (x) == SYMBOL_REF ? SYMBOL_REF_TLS_MODEL (x) : 0;
  if (log)
    return legitimize_tls_address (x, (enum tls_model) log, false);
  if (GET_CODE (x) == CONST
      && GET_CODE (XEXP (x, 0)) == PLUS
      && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
      && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
    {
      rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
				      (enum tls_model) log, false);
      return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
    }

  if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
    {
      rtx tmp = legitimize_pe_coff_symbol (x, true);
      if (tmp)
        return tmp;
    }

  if (flag_pic && SYMBOLIC_CONST (x))
    return legitimize_pic_address (x, 0);

#if TARGET_MACHO
  if (MACHO_DYNAMIC_NO_PIC_P && SYMBOLIC_CONST (x))
    return machopic_indirect_data_reference (x, 0);
#endif

  /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
  if (GET_CODE (x) == ASHIFT
      && CONST_INT_P (XEXP (x, 1))
      && (unsigned HOST_WIDE_INT) INTVAL (XEXP (x, 1)) < 4)
    {
      changed = true;
      log = INTVAL (XEXP (x, 1));
      x = gen_rtx_MULT (Pmode, force_reg (Pmode, XEXP (x, 0)),
			GEN_INT (1 << log));
    }

  if (GET_CODE (x) == PLUS)
    {
      /* Canonicalize shifts by 0, 1, 2, 3 into multiply.  */

      if (GET_CODE (XEXP (x, 0)) == ASHIFT
	  && CONST_INT_P (XEXP (XEXP (x, 0), 1))
	  && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 0), 1)) < 4)
	{
	  changed = true;
	  log = INTVAL (XEXP (XEXP (x, 0), 1));
	  XEXP (x, 0) = gen_rtx_MULT (Pmode,
				      force_reg (Pmode, XEXP (XEXP (x, 0), 0)),
				      GEN_INT (1 << log));
	}

      if (GET_CODE (XEXP (x, 1)) == ASHIFT
	  && CONST_INT_P (XEXP (XEXP (x, 1), 1))
	  && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 1), 1)) < 4)
	{
	  changed = true;
	  log = INTVAL (XEXP (XEXP (x, 1), 1));
	  XEXP (x, 1) = gen_rtx_MULT (Pmode,
				      force_reg (Pmode, XEXP (XEXP (x, 1), 0)),
				      GEN_INT (1 << log));
	}

      /* Put multiply first if it isn't already.  */
      if (GET_CODE (XEXP (x, 1)) == MULT)
	{
	  std::swap (XEXP (x, 0), XEXP (x, 1));
	  changed = true;
	}

      /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
	 into (plus (plus (mult (reg) (const)) (reg)) (const)).  This can be
	 created by virtual register instantiation, register elimination, and
	 similar optimizations.  */
      if (GET_CODE (XEXP (x, 0)) == MULT && GET_CODE (XEXP (x, 1)) == PLUS)
	{
	  changed = true;
	  x = gen_rtx_PLUS (Pmode,
			    gen_rtx_PLUS (Pmode, XEXP (x, 0),
					  XEXP (XEXP (x, 1), 0)),
			    XEXP (XEXP (x, 1), 1));
	}

      /* Canonicalize
	 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
	 into (plus (plus (mult (reg) (const)) (reg)) (const)).  */
      else if (GET_CODE (x) == PLUS && GET_CODE (XEXP (x, 0)) == PLUS
	       && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
	       && GET_CODE (XEXP (XEXP (x, 0), 1)) == PLUS
	       && CONSTANT_P (XEXP (x, 1)))
	{
	  rtx constant;
	  rtx other = NULL_RTX;

	  if (CONST_INT_P (XEXP (x, 1)))
	    {
	      constant = XEXP (x, 1);
	      other = XEXP (XEXP (XEXP (x, 0), 1), 1);
	    }
	  else if (CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 1), 1)))
	    {
	      constant = XEXP (XEXP (XEXP (x, 0), 1), 1);
	      other = XEXP (x, 1);
	    }
	  else
	    constant = 0;

	  if (constant)
	    {
	      changed = true;
	      x = gen_rtx_PLUS (Pmode,
				gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
					      XEXP (XEXP (XEXP (x, 0), 1), 0)),
				plus_constant (Pmode, other,
					       INTVAL (constant)));
	    }
	}

      if (changed && ix86_legitimate_address_p (mode, x, false))
	return x;

      if (GET_CODE (XEXP (x, 0)) == MULT)
	{
	  changed = true;
	  XEXP (x, 0) = copy_addr_to_reg (XEXP (x, 0));
	}

      if (GET_CODE (XEXP (x, 1)) == MULT)
	{
	  changed = true;
	  XEXP (x, 1) = copy_addr_to_reg (XEXP (x, 1));
	}

      if (changed
	  && REG_P (XEXP (x, 1))
	  && REG_P (XEXP (x, 0)))
	return x;

      if (flag_pic && SYMBOLIC_CONST (XEXP (x, 1)))
	{
	  changed = true;
	  x = legitimize_pic_address (x, 0);
	}

      if (changed && ix86_legitimate_address_p (mode, x, false))
	return x;

      if (REG_P (XEXP (x, 0)))
	{
	  rtx temp = gen_reg_rtx (Pmode);
	  rtx val  = force_operand (XEXP (x, 1), temp);
	  if (val != temp)
	    {
	      val = convert_to_mode (Pmode, val, 1);
	      emit_move_insn (temp, val);
	    }

	  XEXP (x, 1) = temp;
	  return x;
	}

      else if (REG_P (XEXP (x, 1)))
	{
	  rtx temp = gen_reg_rtx (Pmode);
	  rtx val  = force_operand (XEXP (x, 0), temp);
	  if (val != temp)
	    {
	      val = convert_to_mode (Pmode, val, 1);
	      emit_move_insn (temp, val);
	    }

	  XEXP (x, 0) = temp;
	  return x;
	}
    }

  return x;
}

/* Print an integer constant expression in assembler syntax.  Addition
   and subtraction are the only arithmetic that may appear in these
   expressions.  FILE is the stdio stream to write to, X is the rtx, and
   CODE is the operand print code from the output string.  */

static void
output_pic_addr_const (FILE *file, rtx x, int code)
{
  char buf[256];

  switch (GET_CODE (x))
    {
    case PC:
      gcc_assert (flag_pic);
      putc ('.', file);
      break;

    case SYMBOL_REF:
      if (TARGET_64BIT || ! TARGET_MACHO_BRANCH_ISLANDS)
	output_addr_const (file, x);
      else
	{
	  const char *name = XSTR (x, 0);

	  /* Mark the decl as referenced so that cgraph will
	     output the function.  */
	  if (SYMBOL_REF_DECL (x))
	    mark_decl_referenced (SYMBOL_REF_DECL (x));

#if TARGET_MACHO
	  if (MACHOPIC_INDIRECT
	      && machopic_classify_symbol (x) == MACHOPIC_UNDEFINED_FUNCTION)
	    name = machopic_indirection_name (x, /*stub_p=*/true);
#endif
	  assemble_name (file, name);
	}
      if (!TARGET_MACHO && !(TARGET_64BIT && TARGET_PECOFF)
	  && code == 'P' && ! SYMBOL_REF_LOCAL_P (x))
	fputs ("@PLT", file);
      break;

    case LABEL_REF:
      x = XEXP (x, 0);
      /* FALLTHRU */
    case CODE_LABEL:
      ASM_GENERATE_INTERNAL_LABEL (buf, "L", CODE_LABEL_NUMBER (x));
      assemble_name (asm_out_file, buf);
      break;

    case CONST_INT:
      fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
      break;

    case CONST:
      /* This used to output parentheses around the expression,
	 but that does not work on the 386 (either ATT or BSD assembler).  */
      output_pic_addr_const (file, XEXP (x, 0), code);
      break;

    case CONST_DOUBLE:
      /* We can't handle floating point constants;
	 TARGET_PRINT_OPERAND must handle them.  */
      output_operand_lossage ("floating constant misused");
      break;

    case PLUS:
      /* Some assemblers need integer constants to appear first.  */
      if (CONST_INT_P (XEXP (x, 0)))
	{
	  output_pic_addr_const (file, XEXP (x, 0), code);
	  putc ('+', file);
	  output_pic_addr_const (file, XEXP (x, 1), code);
	}
      else
	{
	  gcc_assert (CONST_INT_P (XEXP (x, 1)));
	  output_pic_addr_const (file, XEXP (x, 1), code);
	  putc ('+', file);
	  output_pic_addr_const (file, XEXP (x, 0), code);
	}
      break;

    case MINUS:
      if (!TARGET_MACHO)
	putc (ASSEMBLER_DIALECT == ASM_INTEL ? '(' : '[', file);
      output_pic_addr_const (file, XEXP (x, 0), code);
      putc ('-', file);
      output_pic_addr_const (file, XEXP (x, 1), code);
      if (!TARGET_MACHO)
	putc (ASSEMBLER_DIALECT == ASM_INTEL ? ')' : ']', file);
      break;

     case UNSPEC:
       if (XINT (x, 1) == UNSPEC_STACK_CHECK)
	 {
	   bool f = i386_asm_output_addr_const_extra (file, x);
	   gcc_assert (f);
	   break;
	 }

       gcc_assert (XVECLEN (x, 0) == 1);
       output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
       switch (XINT (x, 1))
	{
	case UNSPEC_GOT:
	  fputs ("@GOT", file);
	  break;
	case UNSPEC_GOTOFF:
	  fputs ("@GOTOFF", file);
	  break;
	case UNSPEC_PLTOFF:
	  fputs ("@PLTOFF", file);
	  break;
	case UNSPEC_PCREL:
	  fputs (ASSEMBLER_DIALECT == ASM_ATT ?
		 "(%rip)" : "[rip]", file);
	  break;
	case UNSPEC_GOTPCREL:
	  fputs (ASSEMBLER_DIALECT == ASM_ATT ?
		 "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file);
	  break;
	case UNSPEC_GOTTPOFF:
	  /* FIXME: This might be @TPOFF in Sun ld too.  */
	  fputs ("@gottpoff", file);
	  break;
	case UNSPEC_TPOFF:
	  fputs ("@tpoff", file);
	  break;
	case UNSPEC_NTPOFF:
	  if (TARGET_64BIT)
	    fputs ("@tpoff", file);
	  else
	    fputs ("@ntpoff", file);
	  break;
	case UNSPEC_DTPOFF:
	  fputs ("@dtpoff", file);
	  break;
	case UNSPEC_GOTNTPOFF:
	  if (TARGET_64BIT)
	    fputs (ASSEMBLER_DIALECT == ASM_ATT ?
		   "@gottpoff(%rip)": "@gottpoff[rip]", file);
	  else
	    fputs ("@gotntpoff", file);
	  break;
	case UNSPEC_INDNTPOFF:
	  fputs ("@indntpoff", file);
	  break;
#if TARGET_MACHO
	case UNSPEC_MACHOPIC_OFFSET:
	  putc ('-', file);
	  machopic_output_function_base_name (file);
	  break;
#endif
	default:
	  output_operand_lossage ("invalid UNSPEC as operand");
	  break;
	}
       break;

    default:
      output_operand_lossage ("invalid expression as operand");
    }
}

/* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
   We need to emit DTP-relative relocations.  */

static void ATTRIBUTE_UNUSED
i386_output_dwarf_dtprel (FILE *file, int size, rtx x)
{
  fputs (ASM_LONG, file);
  output_addr_const (file, x);
  fputs ("@dtpoff", file);
  switch (size)
    {
    case 4:
      break;
    case 8:
      fputs (", 0", file);
      break;
    default:
      gcc_unreachable ();
   }
}

/* Return true if X is a representation of the PIC register.  This copes
   with calls from ix86_find_base_term, where the register might have
   been replaced by a cselib value.  */

static bool
ix86_pic_register_p (rtx x)
{
  if (GET_CODE (x) == VALUE && CSELIB_VAL_PTR (x))
    return (pic_offset_table_rtx
	    && rtx_equal_for_cselib_p (x, pic_offset_table_rtx));
  else if (!REG_P (x))
    return false;
  else if (pic_offset_table_rtx)
    {
      if (REGNO (x) == REGNO (pic_offset_table_rtx))
	return true;
      if (HARD_REGISTER_P (x)
	  && !HARD_REGISTER_P (pic_offset_table_rtx)
	  && ORIGINAL_REGNO (x) == REGNO (pic_offset_table_rtx))
	return true;
      return false;
    }
  else
    return REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
}

/* Helper function for ix86_delegitimize_address.
   Attempt to delegitimize TLS local-exec accesses.  */

static rtx
ix86_delegitimize_tls_address (rtx orig_x)
{
  rtx x = orig_x, unspec;
  struct ix86_address addr;

  if (!TARGET_TLS_DIRECT_SEG_REFS)
    return orig_x;
  if (MEM_P (x))
    x = XEXP (x, 0);
  if (GET_CODE (x) != PLUS || GET_MODE (x) != Pmode)
    return orig_x;
  if (ix86_decompose_address (x, &addr) == 0
      || addr.seg != DEFAULT_TLS_SEG_REG
      || addr.disp == NULL_RTX
      || GET_CODE (addr.disp) != CONST)
    return orig_x;
  unspec = XEXP (addr.disp, 0);
  if (GET_CODE (unspec) == PLUS && CONST_INT_P (XEXP (unspec, 1)))
    unspec = XEXP (unspec, 0);
  if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
    return orig_x;
  x = XVECEXP (unspec, 0, 0);
  gcc_assert (GET_CODE (x) == SYMBOL_REF);
  if (unspec != XEXP (addr.disp, 0))
    x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
  if (addr.index)
    {
      rtx idx = addr.index;
      if (addr.scale != 1)
	idx = gen_rtx_MULT (Pmode, idx, GEN_INT (addr.scale));
      x = gen_rtx_PLUS (Pmode, idx, x);
    }
  if (addr.base)
    x = gen_rtx_PLUS (Pmode, addr.base, x);
  if (MEM_P (orig_x))
    x = replace_equiv_address_nv (orig_x, x);
  return x;
}

/* In the name of slightly smaller debug output, and to cater to
   general assembler lossage, recognize PIC+GOTOFF and turn it back
   into a direct symbol reference.

   On Darwin, this is necessary to avoid a crash, because Darwin
   has a different PIC label for each routine but the DWARF debugging
   information is not associated with any particular routine, so it's
   necessary to remove references to the PIC label from RTL stored by
   the DWARF output code.

   This helper is used in the normal ix86_delegitimize_address
   entrypoint (e.g. used in the target delegitimization hook) and
   in ix86_find_base_term.  As compile time memory optimization, we
   avoid allocating rtxes that will not change anything on the outcome
   of the callers (find_base_value and find_base_term).  */

static inline rtx
ix86_delegitimize_address_1 (rtx x, bool base_term_p)
{
  rtx orig_x = delegitimize_mem_from_attrs (x);
  /* addend is NULL or some rtx if x is something+GOTOFF where
     something doesn't include the PIC register.  */
  rtx addend = NULL_RTX;
  /* reg_addend is NULL or a multiple of some register.  */
  rtx reg_addend = NULL_RTX;
  /* const_addend is NULL or a const_int.  */
  rtx const_addend = NULL_RTX;
  /* This is the result, or NULL.  */
  rtx result = NULL_RTX;

  x = orig_x;

  if (MEM_P (x))
    x = XEXP (x, 0);

  if (TARGET_64BIT)
    {
      if (GET_CODE (x) == CONST
          && GET_CODE (XEXP (x, 0)) == PLUS
          && GET_MODE (XEXP (x, 0)) == Pmode
          && CONST_INT_P (XEXP (XEXP (x, 0), 1))
          && GET_CODE (XEXP (XEXP (x, 0), 0)) == UNSPEC
          && XINT (XEXP (XEXP (x, 0), 0), 1) == UNSPEC_PCREL)
        {
	  /* find_base_{value,term} only care about MEMs with arg_pointer_rtx
	     base.  A CONST can't be arg_pointer_rtx based.  */
	  if (base_term_p && MEM_P (orig_x))
	    return orig_x;
	  rtx x2 = XVECEXP (XEXP (XEXP (x, 0), 0), 0, 0);
	  x = gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 1), x2);
	  if (MEM_P (orig_x))
	    x = replace_equiv_address_nv (orig_x, x);
	  return x;
	}

      if (GET_CODE (x) == CONST
	  && GET_CODE (XEXP (x, 0)) == UNSPEC
	  && (XINT (XEXP (x, 0), 1) == UNSPEC_GOTPCREL
	      || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL)
	  && (MEM_P (orig_x) || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL))
	{
	  x = XVECEXP (XEXP (x, 0), 0, 0);
	  if (GET_MODE (orig_x) != GET_MODE (x) && MEM_P (orig_x))
	    {
	      x = lowpart_subreg (GET_MODE (orig_x), x, GET_MODE (x));
	      if (x == NULL_RTX)
		return orig_x;
	    }
	  return x;
	}

      if (ix86_cmodel != CM_MEDIUM_PIC && ix86_cmodel != CM_LARGE_PIC)
	return ix86_delegitimize_tls_address (orig_x);

      /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
	 and -mcmodel=medium -fpic.  */
    }

  if (GET_CODE (x) != PLUS
      || GET_CODE (XEXP (x, 1)) != CONST)
    return ix86_delegitimize_tls_address (orig_x);

  if (ix86_pic_register_p (XEXP (x, 0)))
    /* %ebx + GOT/GOTOFF */
    ;
  else if (GET_CODE (XEXP (x, 0)) == PLUS)
    {
      /* %ebx + %reg * scale + GOT/GOTOFF */
      reg_addend = XEXP (x, 0);
      if (ix86_pic_register_p (XEXP (reg_addend, 0)))
	reg_addend = XEXP (reg_addend, 1);
      else if (ix86_pic_register_p (XEXP (reg_addend, 1)))
	reg_addend = XEXP (reg_addend, 0);
      else
	{
	  reg_addend = NULL_RTX;
	  addend = XEXP (x, 0);
	}
    }
  else
    addend = XEXP (x, 0);

  x = XEXP (XEXP (x, 1), 0);
  if (GET_CODE (x) == PLUS
      && CONST_INT_P (XEXP (x, 1)))
    {
      const_addend = XEXP (x, 1);
      x = XEXP (x, 0);
    }

  if (GET_CODE (x) == UNSPEC
      && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x) && !addend)
	  || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x))
	  || (XINT (x, 1) == UNSPEC_PLTOFF && ix86_cmodel == CM_LARGE_PIC
	      && !MEM_P (orig_x) && !addend)))
    result = XVECEXP (x, 0, 0);

  if (!TARGET_64BIT && TARGET_MACHO && darwin_local_data_pic (x)
      && !MEM_P (orig_x))
    result = XVECEXP (x, 0, 0);

  if (! result)
    return ix86_delegitimize_tls_address (orig_x);

  /* For (PLUS something CONST_INT) both find_base_{value,term} just
     recurse on the first operand.  */
  if (const_addend && !base_term_p)
    result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
  if (reg_addend)
    result = gen_rtx_PLUS (Pmode, reg_addend, result);
  if (addend)
    {
      /* If the rest of original X doesn't involve the PIC register, add
	 addend and subtract pic_offset_table_rtx.  This can happen e.g.
	 for code like:
	 leal (%ebx, %ecx, 4), %ecx
	 ...
	 movl foo@GOTOFF(%ecx), %edx
	 in which case we return (%ecx - %ebx) + foo
	 or (%ecx - _GLOBAL_OFFSET_TABLE_) + foo if pseudo_pic_reg
	 and reload has completed.  */
      if (pic_offset_table_rtx
	  && (!reload_completed || !ix86_use_pseudo_pic_reg ()))
        result = gen_rtx_PLUS (Pmode, gen_rtx_MINUS (Pmode, copy_rtx (addend),
						     pic_offset_table_rtx),
			       result);
      else if (pic_offset_table_rtx && !TARGET_MACHO && !TARGET_VXWORKS_RTP)
	{
	  rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
	  tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
	  result = gen_rtx_PLUS (Pmode, tmp, result);
	}
      else
	return orig_x;
    }
  if (GET_MODE (orig_x) != Pmode && MEM_P (orig_x))
    {
      result = lowpart_subreg (GET_MODE (orig_x), result, Pmode);
      if (result == NULL_RTX)
	return orig_x;
    }
  return result;
}

/* The normal instantiation of the above template.  */

static rtx
ix86_delegitimize_address (rtx x)
{
  return ix86_delegitimize_address_1 (x, false);
}

/* If X is a machine specific address (i.e. a symbol or label being
   referenced as a displacement from the GOT implemented using an
   UNSPEC), then return the base term.  Otherwise return X.  */

rtx
ix86_find_base_term (rtx x)
{
  rtx term;

  if (TARGET_64BIT)
    {
      if (GET_CODE (x) != CONST)
	return x;
      term = XEXP (x, 0);
      if (GET_CODE (term) == PLUS
	  && CONST_INT_P (XEXP (term, 1)))
	term = XEXP (term, 0);
      if (GET_CODE (term) != UNSPEC
	  || (XINT (term, 1) != UNSPEC_GOTPCREL
	      && XINT (term, 1) != UNSPEC_PCREL))
	return x;

      return XVECEXP (term, 0, 0);
    }

  return ix86_delegitimize_address_1 (x, true);
}

static void
put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
		    bool fp, FILE *file)
{
  const char *suffix;

  if (mode == CCFPmode || mode == CCFPUmode)
    {
      code = ix86_fp_compare_code_to_integer (code);
      mode = CCmode;
    }
  if (reverse)
    code = reverse_condition (code);

  switch (code)
    {
    case EQ:
      switch (mode)
	{
	case CCAmode:
	  suffix = "a";
	  break;
	case CCCmode:
	  suffix = "c";
	  break;
	case CCOmode:
	  suffix = "o";
	  break;
	case CCPmode:
	  suffix = "p";
	  break;
	case CCSmode:
	  suffix = "s";
	  break;
	default:
	  suffix = "e";
	  break;
	}
      break;
    case NE:
      switch (mode)
	{
	case CCAmode:
	  suffix = "na";
	  break;
	case CCCmode:
	  suffix = "nc";
	  break;
	case CCOmode:
	  suffix = "no";
	  break;
	case CCPmode:
	  suffix = "np";
	  break;
	case CCSmode:
	  suffix = "ns";
	  break;
	default:
	  suffix = "ne";
	  break;
	}
      break;
    case GT:
      gcc_assert (mode == CCmode || mode == CCNOmode || mode == CCGCmode);
      suffix = "g";
      break;
    case GTU:
      /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
	 Those same assemblers have the same but opposite lossage on cmov.  */
      if (mode == CCmode)
	suffix = fp ? "nbe" : "a";
      else
	gcc_unreachable ();
      break;
    case LT:
      switch (mode)
	{
	case CCNOmode:
	case CCGOCmode:
	  suffix = "s";
	  break;

	case CCmode:
	case CCGCmode:
	  suffix = "l";
	  break;

	default:
	  gcc_unreachable ();
	}
      break;
    case LTU:
      if (mode == CCmode)
	suffix = "b";
      else if (mode == CCCmode)
	suffix = fp ? "b" : "c";
      else
	gcc_unreachable ();
      break;
    case GE:
      switch (mode)
	{
	case CCNOmode:
	case CCGOCmode:
	  suffix = "ns";
	  break;

	case CCmode:
	case CCGCmode:
	  suffix = "ge";
	  break;

	default:
	  gcc_unreachable ();
	}
      break;
    case GEU:
      if (mode == CCmode)
	suffix = "nb";
      else if (mode == CCCmode)
	suffix = fp ? "nb" : "nc";
      else
	gcc_unreachable ();
      break;
    case LE:
      gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
      suffix = "le";
      break;
    case LEU:
      if (mode == CCmode)
	suffix = "be";
      else
	gcc_unreachable ();
      break;
    case UNORDERED:
      suffix = fp ? "u" : "p";
      break;
    case ORDERED:
      suffix = fp ? "nu" : "np";
      break;
    default:
      gcc_unreachable ();
    }
  fputs (suffix, file);
}

/* Print the name of register X to FILE based on its machine mode and number.
   If CODE is 'w', pretend the mode is HImode.
   If CODE is 'b', pretend the mode is QImode.
   If CODE is 'k', pretend the mode is SImode.
   If CODE is 'q', pretend the mode is DImode.
   If CODE is 'x', pretend the mode is V4SFmode.
   If CODE is 't', pretend the mode is V8SFmode.
   If CODE is 'g', pretend the mode is V16SFmode.
   If CODE is 'h', pretend the reg is the 'high' byte register.
   If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
   If CODE is 'd', duplicate the operand for AVX instruction.
 */

void
print_reg (rtx x, int code, FILE *file)
{
  const char *reg;
  int msize;
  unsigned int regno;
  bool duplicated;

  if (ASSEMBLER_DIALECT == ASM_ATT)
    putc ('%', file);

  if (x == pc_rtx)
    {
      gcc_assert (TARGET_64BIT);
      fputs ("rip", file);
      return;
    }

  if (code == 'y' && STACK_TOP_P (x))
    {
      fputs ("st(0)", file);
      return;
    }

  if (code == 'w')
    msize = 2;
  else if (code == 'b')
    msize = 1;
  else if (code == 'k')
    msize = 4;
  else if (code == 'q')
    msize = 8;
  else if (code == 'h')
    msize = 0;
  else if (code == 'x')
    msize = 16;
  else if (code == 't')
    msize = 32;
  else if (code == 'g')
    msize = 64;
  else
    msize = GET_MODE_SIZE (GET_MODE (x));

  regno = REGNO (x);

  gcc_assert (regno != ARG_POINTER_REGNUM
	      && regno != FRAME_POINTER_REGNUM
	      && regno != FPSR_REG
	      && regno != FPCR_REG);

  if (regno == FLAGS_REG)
    {
      output_operand_lossage ("invalid use of asm flag output");
      return;
    }

  duplicated = code == 'd' && TARGET_AVX;

  switch (msize)
    {
    case 16:
    case 12:
    case 8:
      if (GENERAL_REGNO_P (regno) && msize > GET_MODE_SIZE (word_mode))
	warning (0, "unsupported size for integer register");
      /* FALLTHRU */
    case 4:
      if (LEGACY_INT_REGNO_P (regno))
	putc (msize > 4 && TARGET_64BIT ? 'r' : 'e', file);
      /* FALLTHRU */
    case 2:
    normal:
      reg = hi_reg_name[regno];
      break;
    case 1:
      if (regno >= ARRAY_SIZE (qi_reg_name))
	goto normal;
      if (!ANY_QI_REGNO_P (regno))
	error ("unsupported size for integer register");
      reg = qi_reg_name[regno];
      break;
    case 0:
      if (regno >= ARRAY_SIZE (qi_high_reg_name))
	goto normal;
      reg = qi_high_reg_name[regno];
      break;
    case 32:
    case 64:
      if (SSE_REGNO_P (regno))
	{
	  gcc_assert (!duplicated);
	  putc (msize == 32 ? 'y' : 'z', file);
	  reg = hi_reg_name[regno] + 1;
	  break;
	}
      goto normal;
    default:
      gcc_unreachable ();
    }

  fputs (reg, file);

  /* Irritatingly, AMD extended registers use
     different naming convention: "r%d[bwd]"  */
  if (REX_INT_REGNO_P (regno))
    {
      gcc_assert (TARGET_64BIT);
      switch (msize)
	{
	  case 0:
	    error ("extended registers have no high halves");
	    break;
	  case 1:
	    putc ('b', file);
	    break;
	  case 2:
	    putc ('w', file);
	    break;
	  case 4:
	    putc ('d', file);
	    break;
	  case 8:
	    /* no suffix */
	    break;
	  default:
	    error ("unsupported operand size for extended register");
	    break;
	}
      return;
    }

  if (duplicated)
    {
      if (ASSEMBLER_DIALECT == ASM_ATT)
	fprintf (file, ", %%%s", reg);
      else
	fprintf (file, ", %s", reg);
    }
}

/* Meaning of CODE:
   L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
   C -- print opcode suffix for set/cmov insn.
   c -- like C, but print reversed condition
   F,f -- likewise, but for floating-point.
   O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
	otherwise nothing
   R -- print embeded rounding and sae.
   r -- print only sae.
   z -- print the opcode suffix for the size of the current operand.
   Z -- likewise, with special suffixes for x87 instructions.
   * -- print a star (in certain assembler syntax)
   A -- print an absolute memory reference.
   E -- print address with DImode register names if TARGET_64BIT.
   w -- print the operand as if it's a "word" (HImode) even if it isn't.
   s -- print a shift double count, followed by the assemblers argument
	delimiter.
   b -- print the QImode name of the register for the indicated operand.
	%b0 would print %al if operands[0] is reg 0.
   w --  likewise, print the HImode name of the register.
   k --  likewise, print the SImode name of the register.
   q --  likewise, print the DImode name of the register.
   x --  likewise, print the V4SFmode name of the register.
   t --  likewise, print the V8SFmode name of the register.
   g --  likewise, print the V16SFmode name of the register.
   h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
   y -- print "st(0)" instead of "st" as a register.
   d -- print duplicated register operand for AVX instruction.
   D -- print condition for SSE cmp instruction.
   P -- if PIC, print an @PLT suffix.
   p -- print raw symbol name.
   X -- don't print any sort of PIC '@' suffix for a symbol.
   & -- print some in-use local-dynamic symbol name.
   H -- print a memory address offset by 8; used for sse high-parts
   Y -- print condition for XOP pcom* instruction.
   + -- print a branch hint as 'cs' or 'ds' prefix
   ; -- print a semicolon (after prefixes due to bug in older gas).
   ~ -- print "i" if TARGET_AVX2, "f" otherwise.
   @ -- print a segment register of thread base pointer load
   ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
   ! -- print MPX prefix for jxx/call/ret instructions if required.
 */

void
ix86_print_operand (FILE *file, rtx x, int code)
{
  if (code)
    {
      switch (code)
	{
	case 'A':
	  switch (ASSEMBLER_DIALECT)
	    {
	    case ASM_ATT:
	      putc ('*', file);
	      break;

	    case ASM_INTEL:
	      /* Intel syntax. For absolute addresses, registers should not
		 be surrounded by braces.  */
	      if (!REG_P (x))
		{
		  putc ('[', file);
		  ix86_print_operand (file, x, 0);
		  putc (']', file);
		  return;
		}
	      break;

	    default:
	      gcc_unreachable ();
	    }

	  ix86_print_operand (file, x, 0);
	  return;

	case 'E':
	  /* Wrap address in an UNSPEC to declare special handling.  */
	  if (TARGET_64BIT)
	    x = gen_rtx_UNSPEC (DImode, gen_rtvec (1, x), UNSPEC_LEA_ADDR);

	  output_address (VOIDmode, x);
	  return;

	case 'L':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('l', file);
	  return;

	case 'W':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('w', file);
	  return;

	case 'B':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('b', file);
	  return;

	case 'Q':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('l', file);
	  return;

	case 'S':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('s', file);
	  return;

	case 'T':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('t', file);
	  return;

	case 'O':
#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
	  if (ASSEMBLER_DIALECT != ASM_ATT)
	    return;

	  switch (GET_MODE_SIZE (GET_MODE (x)))
	    {
	    case 2:
	      putc ('w', file);
	      break;
  
	    case 4:
	      putc ('l', file);
	      break;

	    case 8:
	      putc ('q', file);
	      break;

	    default:
	      output_operand_lossage ("invalid operand size for operand "
				      "code 'O'");
	      return;
	    }

	  putc ('.', file);
#endif
	  return;

	case 'z':
	  if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
	    {
	      /* Opcodes don't get size suffixes if using Intel opcodes.  */
	      if (ASSEMBLER_DIALECT == ASM_INTEL)
		return;

	      switch (GET_MODE_SIZE (GET_MODE (x)))
		{
		case 1:
		  putc ('b', file);
		  return;

		case 2:
		  putc ('w', file);
		  return;

		case 4:
		  putc ('l', file);
		  return;

		case 8:
		  putc ('q', file);
		  return;

		default:
		  output_operand_lossage ("invalid operand size for operand "
					  "code 'z'");
		  return;
		}
	    }

	  if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
	    warning (0, "non-integer operand used with operand code 'z'");
	  /* FALLTHRU */

	case 'Z':
	  /* 387 opcodes don't get size suffixes if using Intel opcodes.  */
	  if (ASSEMBLER_DIALECT == ASM_INTEL)
	    return;

	  if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
	    {
	      switch (GET_MODE_SIZE (GET_MODE (x)))
		{
		case 2:
#ifdef HAVE_AS_IX86_FILDS
		  putc ('s', file);
#endif
		  return;

		case 4:
		  putc ('l', file);
		  return;

		case 8:
#ifdef HAVE_AS_IX86_FILDQ
		  putc ('q', file);
#else
		  fputs ("ll", file);
#endif
		  return;

		default:
		  break;
		}
	    }
	  else if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
	    {
	      /* 387 opcodes don't get size suffixes
		 if the operands are registers.  */
	      if (STACK_REG_P (x))
		return;

	      switch (GET_MODE_SIZE (GET_MODE (x)))
		{
		case 4:
		  putc ('s', file);
		  return;

		case 8:
		  putc ('l', file);
		  return;

		case 12:
		case 16:
		  putc ('t', file);
		  return;

		default:
		  break;
		}
	    }
	  else
	    {
	      output_operand_lossage ("invalid operand type used with "
				      "operand code 'Z'");
	      return;
	    }

	  output_operand_lossage ("invalid operand size for operand code 'Z'");
	  return;

	case 'd':
	case 'b':
	case 'w':
	case 'k':
	case 'q':
	case 'h':
	case 't':
	case 'g':
	case 'y':
	case 'x':
	case 'X':
	case 'P':
	case 'p':
	  break;

	case 's':
	  if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT)
	    {
	      ix86_print_operand (file, x, 0);
	      fputs (", ", file);
	    }
	  return;

	case 'Y':
	  switch (GET_CODE (x))
	    {
	    case NE:
	      fputs ("neq", file);
	      break;
	    case EQ:
	      fputs ("eq", file);
	      break;
	    case GE:
	    case GEU:
	      fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "ge" : "unlt", file);
	      break;
	    case GT:
	    case GTU:
	      fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "gt" : "unle", file);
	      break;
	    case LE:
	    case LEU:
	      fputs ("le", file);
	      break;
	    case LT:
	    case LTU:
	      fputs ("lt", file);
	      break;
	    case UNORDERED:
	      fputs ("unord", file);
	      break;
	    case ORDERED:
	      fputs ("ord", file);
	      break;
	    case UNEQ:
	      fputs ("ueq", file);
	      break;
	    case UNGE:
	      fputs ("nlt", file);
	      break;
	    case UNGT:
	      fputs ("nle", file);
	      break;
	    case UNLE:
	      fputs ("ule", file);
	      break;
	    case UNLT:
	      fputs ("ult", file);
	      break;
	    case LTGT:
	      fputs ("une", file);
	      break;
	    default:
	      output_operand_lossage ("operand is not a condition code, "
				      "invalid operand code 'Y'");
	      return;
	    }
	  return;

	case 'D':
	  /* Little bit of braindamage here.  The SSE compare instructions
	     does use completely different names for the comparisons that the
	     fp conditional moves.  */
	  switch (GET_CODE (x))
	    {
	    case UNEQ:
	      if (TARGET_AVX)
		{
		  fputs ("eq_us", file);
		  break;
		}
	     /* FALLTHRU */
	    case EQ:
	      fputs ("eq", file);
	      break;
	    case UNLT:
	      if (TARGET_AVX)
		{
		  fputs ("nge", file);
		  break;
		}
	     /* FALLTHRU */
	    case LT:
	      fputs ("lt", file);
	      break;
	    case UNLE:
	      if (TARGET_AVX)
		{
		  fputs ("ngt", file);
		  break;
		}
	     /* FALLTHRU */
	    case LE:
	      fputs ("le", file);
	      break;
	    case UNORDERED:
	      fputs ("unord", file);
	      break;
	    case LTGT:
	      if (TARGET_AVX)
		{
		  fputs ("neq_oq", file);
		  break;
		}
	     /* FALLTHRU */
	    case NE:
	      fputs ("neq", file);
	      break;
	    case GE:
	      if (TARGET_AVX)
		{
		  fputs ("ge", file);
		  break;
		}
	     /* FALLTHRU */
	    case UNGE:
	      fputs ("nlt", file);
	      break;
	    case GT:
	      if (TARGET_AVX)
		{
		  fputs ("gt", file);
		  break;
		}
	     /* FALLTHRU */
	    case UNGT:
	      fputs ("nle", file);
	      break;
	    case ORDERED:
	      fputs ("ord", file);
	      break;
	    default:
	      output_operand_lossage ("operand is not a condition code, "
				      "invalid operand code 'D'");
	      return;
	    }
	  return;

	case 'F':
	case 'f':
#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('.', file);
	  gcc_fallthrough ();
#endif

	case 'C':
	case 'c':
	  if (!COMPARISON_P (x))
	    {
	      output_operand_lossage ("operand is not a condition code, "
				      "invalid operand code '%c'", code);
	      return;
	    }
	  put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)),
			      code == 'c' || code == 'f',
			      code == 'F' || code == 'f',
			      file);
	  return;

	case 'H':
	  if (!offsettable_memref_p (x))
	    {
	      output_operand_lossage ("operand is not an offsettable memory "
				      "reference, invalid operand code 'H'");
	      return;
	    }
	  /* It doesn't actually matter what mode we use here, as we're
	     only going to use this for printing.  */
	  x = adjust_address_nv (x, DImode, 8);
	  /* Output 'qword ptr' for intel assembler dialect.  */
	  if (ASSEMBLER_DIALECT == ASM_INTEL)
	    code = 'q';
	  break;

	case 'K':
	  if (!CONST_INT_P (x))
	    {
	      output_operand_lossage ("operand is not an integer, invalid "
				      "operand code 'K'");
	      return;
	    }

	  if (INTVAL (x) & IX86_HLE_ACQUIRE)
#ifdef HAVE_AS_IX86_HLE
	    fputs ("xacquire ", file);
#else
	    fputs ("\n" ASM_BYTE "0xf2\n\t", file);
#endif
	  else if (INTVAL (x) & IX86_HLE_RELEASE)
#ifdef HAVE_AS_IX86_HLE
	    fputs ("xrelease ", file);
#else
	    fputs ("\n" ASM_BYTE "0xf3\n\t", file);
#endif
	  /* We do not want to print value of the operand.  */
	  return;

	case 'N':
	  if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
	    fputs ("{z}", file);
	  return;

	case 'r':
	  if (!CONST_INT_P (x) || INTVAL (x) != ROUND_SAE)
	    {
	      output_operand_lossage ("operand is not a specific integer, "
				      "invalid operand code 'r'");
	      return;
	    }

	  if (ASSEMBLER_DIALECT == ASM_INTEL)
	    fputs (", ", file);

	  fputs ("{sae}", file);

	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    fputs (", ", file);

	  return;

	case 'R':
	  if (!CONST_INT_P (x))
	    {
	      output_operand_lossage ("operand is not an integer, invalid "
				      "operand code 'R'");
	      return;
	    }

	  if (ASSEMBLER_DIALECT == ASM_INTEL)
	    fputs (", ", file);

	  switch (INTVAL (x))
	    {
	    case ROUND_NEAREST_INT | ROUND_SAE:
	      fputs ("{rn-sae}", file);
	      break;
	    case ROUND_NEG_INF | ROUND_SAE:
	      fputs ("{rd-sae}", file);
	      break;
	    case ROUND_POS_INF | ROUND_SAE:
	      fputs ("{ru-sae}", file);
	      break;
	    case ROUND_ZERO | ROUND_SAE:
	      fputs ("{rz-sae}", file);
	      break;
	    default:
	      output_operand_lossage ("operand is not a specific integer, "
				      "invalid operand code 'R'");
	    }

	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    fputs (", ", file);

	  return;

	case '*':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('*', file);
	  return;

	case '&':
	  {
	    const char *name = get_some_local_dynamic_name ();
	    if (name == NULL)
	      output_operand_lossage ("'%%&' used without any "
				      "local dynamic TLS references");
	    else
	      assemble_name (file, name);
	    return;
	  }

	case '+':
	  {
	    rtx x;

	    if (!optimize
	        || optimize_function_for_size_p (cfun)
		|| !TARGET_BRANCH_PREDICTION_HINTS)
	      return;

	    x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
	    if (x)
	      {
		int pred_val = XINT (x, 0);

		if (pred_val < REG_BR_PROB_BASE * 45 / 100
		    || pred_val > REG_BR_PROB_BASE * 55 / 100)
		  {
		    bool taken = pred_val > REG_BR_PROB_BASE / 2;
		    bool cputaken
		      = final_forward_branch_p (current_output_insn) == 0;

		    /* Emit hints only in the case default branch prediction
		       heuristics would fail.  */
		    if (taken != cputaken)
		      {
			/* We use 3e (DS) prefix for taken branches and
			   2e (CS) prefix for not taken branches.  */
			if (taken)
			  fputs ("ds ; ", file);
			else
			  fputs ("cs ; ", file);
		      }
		  }
	      }
	    return;
	  }

	case ';':
#ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
	  putc (';', file);
#endif
	  return;

	case '@':
	  if (ASSEMBLER_DIALECT == ASM_ATT)
	    putc ('%', file);

	  /* The kernel uses a different segment register for performance
	     reasons; a system call would not have to trash the userspace
	     segment register, which would be expensive.  */
	  if (TARGET_64BIT && ix86_cmodel != CM_KERNEL)
	    fputs ("fs", file);
	  else
	    fputs ("gs", file);
	  return;

	case '~':
	  putc (TARGET_AVX2 ? 'i' : 'f', file);
	  return;

	case '^':
	  if (TARGET_64BIT && Pmode != word_mode)
	    fputs ("addr32 ", file);
	  return;

	case '!':
	  if (ix86_bnd_prefixed_insn_p (current_output_insn))
	    fputs ("bnd ", file);
	  return;

	default:
	  output_operand_lossage ("invalid operand code '%c'", code);
	}
    }

  if (REG_P (x))
    print_reg (x, code, file);

  else if (MEM_P (x))
    {
      rtx addr = XEXP (x, 0);

      /* No `byte ptr' prefix for call instructions ... */
      if (ASSEMBLER_DIALECT == ASM_INTEL && code != 'X' && code != 'P')
	{
	  machine_mode mode = GET_MODE (x);
	  const char *size;

	  /* Check for explicit size override codes.  */
	  if (code == 'b')
	    size = "BYTE";
	  else if (code == 'w')
	    size = "WORD";
	  else if (code == 'k')
	    size = "DWORD";
	  else if (code == 'q')
	    size = "QWORD";
	  else if (code == 'x')
	    size = "XMMWORD";
	  else if (code == 't')
	    size = "YMMWORD";
	  else if (code == 'g')
	    size = "ZMMWORD";
	  else if (mode == BLKmode)
	    /* ... or BLKmode operands, when not overridden.  */
	    size = NULL;
	  else
	    switch (GET_MODE_SIZE (mode))
	      {
	      case 1: size = "BYTE"; break;
	      case 2: size = "WORD"; break;
	      case 4: size = "DWORD"; break;
	      case 8: size = "QWORD"; break;
	      case 12: size = "TBYTE"; break;
	      case 16:
		if (mode == XFmode)
		  size = "TBYTE";
		else
		  size = "XMMWORD";
		break;
	      case 32: size = "YMMWORD"; break;
	      case 64: size = "ZMMWORD"; break;
	      default:
		gcc_unreachable ();
	      }
	  if (size)
	    {
	      fputs (size, file);
	      fputs (" PTR ", file);
	    }
	}

      if (this_is_asm_operands && ! address_operand (addr, VOIDmode))
	output_operand_lossage ("invalid constraints for operand");
      else
	ix86_print_operand_address_as
	  (file, addr, MEM_ADDR_SPACE (x), code == 'p' || code == 'P');
    }

  else if (CONST_DOUBLE_P (x) && GET_MODE (x) == SFmode)
    {
      long l;

      REAL_VALUE_TO_TARGET_SINGLE (*CONST_DOUBLE_REAL_VALUE (x), l);

      if (ASSEMBLER_DIALECT == ASM_ATT)
	putc ('$', file);
      /* Sign extend 32bit SFmode immediate to 8 bytes.  */
      if (code == 'q')
	fprintf (file, "0x%08" HOST_LONG_LONG_FORMAT "x",
		 (unsigned long long) (int) l);
      else
	fprintf (file, "0x%08x", (unsigned int) l);
    }

  else if (CONST_DOUBLE_P (x) && GET_MODE (x) == DFmode)
    {
      long l[2];

      REAL_VALUE_TO_TARGET_DOUBLE (*CONST_DOUBLE_REAL_VALUE (x), l);

      if (ASSEMBLER_DIALECT == ASM_ATT)
	putc ('$', file);
      fprintf (file, "0x%lx%08lx", l[1] & 0xffffffff, l[0] & 0xffffffff);
    }

  /* These float cases don't actually occur as immediate operands.  */
  else if (CONST_DOUBLE_P (x) && GET_MODE (x) == XFmode)
    {
      char dstr[30];

      real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
      fputs (dstr, file);
    }

  else
    {
      /* We have patterns that allow zero sets of memory, for instance.
	 In 64-bit mode, we should probably support all 8-byte vectors,
	 since we can in fact encode that into an immediate.  */
      if (GET_CODE (x) == CONST_VECTOR)
	{
	  gcc_assert (x == CONST0_RTX (GET_MODE (x)));
	  x = const0_rtx;
	}

      if (code != 'P' && code != 'p')
	{
	  if (CONST_INT_P (x))
	    {
	      if (ASSEMBLER_DIALECT == ASM_ATT)
		putc ('$', file);
	    }
	  else if (GET_CODE (x) == CONST || GET_CODE (x) == SYMBOL_REF
		   || GET_CODE (x) == LABEL_REF)
	    {
	      if (ASSEMBLER_DIALECT == ASM_ATT)
		putc ('$', file);
	      else
		fputs ("OFFSET FLAT:", file);
	    }
	}
      if (CONST_INT_P (x))
	fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
      else if (flag_pic || MACHOPIC_INDIRECT)
	output_pic_addr_const (file, x, code);
      else
	output_addr_const (file, x);
    }
}

static bool
ix86_print_operand_punct_valid_p (unsigned char code)
{
  return (code == '@' || code == '*' || code == '+' || code == '&'
	  || code == ';' || code == '~' || code == '^' || code == '!');
}

/* Print a memory operand whose address is ADDR.  */

static void
ix86_print_operand_address_as (FILE *file, rtx addr,
			       addr_space_t as, bool no_rip)
{
  struct ix86_address parts;
  rtx base, index, disp;
  int scale;
  int ok;
  bool vsib = false;
  int code = 0;

  if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_VSIBADDR)
    {
      ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
      gcc_assert (parts.index == NULL_RTX);
      parts.index = XVECEXP (addr, 0, 1);
      parts.scale = INTVAL (XVECEXP (addr, 0, 2));
      addr = XVECEXP (addr, 0, 0);
      vsib = true;
    }
  else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_LEA_ADDR)
    {
      gcc_assert (TARGET_64BIT);
      ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
      code = 'q';
    }
  else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDMK_ADDR)
    {
      ok = ix86_decompose_address (XVECEXP (addr, 0, 1), &parts);
      gcc_assert (parts.base == NULL_RTX || parts.index == NULL_RTX);
      if (parts.base != NULL_RTX)
	{
	  parts.index = parts.base;
	  parts.scale = 1;
	}
      parts.base = XVECEXP (addr, 0, 0);
      addr = XVECEXP (addr, 0, 0);
    }
  else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDLDX_ADDR)
    {
      ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
      gcc_assert (parts.index == NULL_RTX);
      parts.index = XVECEXP (addr, 0, 1);
      addr = XVECEXP (addr, 0, 0);
    }
  else
    ok = ix86_decompose_address (addr, &parts);

  gcc_assert (ok);

  base = parts.base;
  index = parts.index;
  disp = parts.disp;
  scale = parts.scale;

  if (ADDR_SPACE_GENERIC_P (as))
    as = parts.seg;
  else
    gcc_assert (ADDR_SPACE_GENERIC_P (parts.seg));

  if (!ADDR_SPACE_GENERIC_P (as))
    {
      const char *string;

      if (as == ADDR_SPACE_SEG_FS)
	string = (ASSEMBLER_DIALECT == ASM_ATT ? "%fs:" : "fs:");
      else if (as == ADDR_SPACE_SEG_GS)
	string = (ASSEMBLER_DIALECT == ASM_ATT ? "%gs:" : "gs:");
      else
	gcc_unreachable ();
      fputs (string, file);
    }

  /* Use one byte shorter RIP relative addressing for 64bit mode.  */
  if (TARGET_64BIT && !base && !index && !no_rip)
    {
      rtx symbol = disp;

      if (GET_CODE (disp) == CONST
	  && GET_CODE (XEXP (disp, 0)) == PLUS
	  && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
	symbol = XEXP (XEXP (disp, 0), 0);

      if (GET_CODE (symbol) == LABEL_REF
	  || (GET_CODE (symbol) == SYMBOL_REF
	      && SYMBOL_REF_TLS_MODEL (symbol) == 0))
	base = pc_rtx;
    }

  if (!base && !index)
    {
      /* Displacement only requires special attention.  */
      if (CONST_INT_P (disp))
	{
	  if (ASSEMBLER_DIALECT == ASM_INTEL && parts.seg == ADDR_SPACE_GENERIC)
	    fputs ("ds:", file);
	  fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (disp));
	}
      /* Load the external function address via the GOT slot to avoid PLT.  */
      else if (GET_CODE (disp) == CONST
	       && GET_CODE (XEXP (disp, 0)) == UNSPEC
	       && (XINT (XEXP (disp, 0), 1) == UNSPEC_GOTPCREL
		   || XINT (XEXP (disp, 0), 1) == UNSPEC_GOT)
	       && ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
	output_pic_addr_const (file, disp, 0);
      else if (flag_pic)
	output_pic_addr_const (file, disp, 0);
      else
	output_addr_const (file, disp);
    }
  else
    {
      /* Print SImode register names to force addr32 prefix.  */
      if (SImode_address_operand (addr, VOIDmode))
	{
	  if (flag_checking)
	    {
	      gcc_assert (TARGET_64BIT);
	      switch (GET_CODE (addr))
		{
		case SUBREG:
		  gcc_assert (GET_MODE (addr) == SImode);
		  gcc_assert (GET_MODE (SUBREG_REG (addr)) == DImode);
		  break;
		case ZERO_EXTEND:
		case AND:
		  gcc_assert (GET_MODE (addr) == DImode);
		  break;
		default:
		  gcc_unreachable ();
		}
	    }
	  gcc_assert (!code);
	  code = 'k';
	}
      else if (code == 0
	       && TARGET_X32
	       && disp
	       && CONST_INT_P (disp)
	       && INTVAL (disp) < -16*1024*1024)
	{
	  /* X32 runs in 64-bit mode, where displacement, DISP, in
	     address DISP(%r64), is encoded as 32-bit immediate sign-
	     extended from 32-bit to 64-bit.  For -0x40000300(%r64),
	     address is %r64 + 0xffffffffbffffd00.  When %r64 <
	     0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
	     which is invalid for x32.  The correct address is %r64
	     - 0x40000300 == 0xf7ffdd64.  To properly encode
	     -0x40000300(%r64) for x32, we zero-extend negative
	     displacement by forcing addr32 prefix which truncates
	     0xfffffffff7ffdd64 to 0xf7ffdd64.  In theory, we should
	     zero-extend all negative displacements, including -1(%rsp).
	     However, for small negative displacements, sign-extension
	     won't cause overflow.  We only zero-extend negative
	     displacements if they < -16*1024*1024, which is also used
	     to check legitimate address displacements for PIC.  */
	  code = 'k';
	}

      if (ASSEMBLER_DIALECT == ASM_ATT)
	{
	  if (disp)
	    {
	      if (flag_pic)
		output_pic_addr_const (file, disp, 0);
	      else if (GET_CODE (disp) == LABEL_REF)
		output_asm_label (disp);
	      else
		output_addr_const (file, disp);
	    }

	  putc ('(', file);
	  if (base)
	    print_reg (base, code, file);
	  if (index)
	    {
	      putc (',', file);
	      print_reg (index, vsib ? 0 : code, file);
	      if (scale != 1 || vsib)
		fprintf (file, ",%d", scale);
	    }
	  putc (')', file);
	}
      else
	{
	  rtx offset = NULL_RTX;

	  if (disp)
	    {
	      /* Pull out the offset of a symbol; print any symbol itself.  */
	      if (GET_CODE (disp) == CONST
		  && GET_CODE (XEXP (disp, 0)) == PLUS
		  && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
		{
		  offset = XEXP (XEXP (disp, 0), 1);
		  disp = gen_rtx_CONST (VOIDmode,
					XEXP (XEXP (disp, 0), 0));
		}

	      if (flag_pic)
		output_pic_addr_const (file, disp, 0);
	      else if (GET_CODE (disp) == LABEL_REF)
		output_asm_label (disp);
	      else if (CONST_INT_P (disp))
		offset = disp;
	      else
		output_addr_const (file, disp);
	    }

	  putc ('[', file);
	  if (base)
	    {
	      print_reg (base, code, file);
	      if (offset)
		{
		  if (INTVAL (offset) >= 0)
		    putc ('+', file);
		  fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
		}
	    }
	  else if (offset)
	    fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
	  else
	    putc ('0', file);

	  if (index)
	    {
	      putc ('+', file);
	      print_reg (index, vsib ? 0 : code, file);
	      if (scale != 1 || vsib)
		fprintf (file, "*%d", scale);
	    }
	  putc (']', file);
	}
    }
}

static void
ix86_print_operand_address (FILE *file, machine_mode /*mode*/, rtx addr)
{
  ix86_print_operand_address_as (file, addr, ADDR_SPACE_GENERIC, false);
}

/* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA.  */

static bool
i386_asm_output_addr_const_extra (FILE *file, rtx x)
{
  rtx op;

  if (GET_CODE (x) != UNSPEC)
    return false;

  op = XVECEXP (x, 0, 0);
  switch (XINT (x, 1))
    {
    case UNSPEC_GOTTPOFF:
      output_addr_const (file, op);
      /* FIXME: This might be @TPOFF in Sun ld.  */
      fputs ("@gottpoff", file);
      break;
    case UNSPEC_TPOFF:
      output_addr_const (file, op);
      fputs ("@tpoff", file);
      break;
    case UNSPEC_NTPOFF:
      output_addr_const (file, op);
      if (TARGET_64BIT)
	fputs ("@tpoff", file);
      else
	fputs ("@ntpoff", file);
      break;
    case UNSPEC_DTPOFF:
      output_addr_const (file, op);
      fputs ("@dtpoff", file);
      break;
    case UNSPEC_GOTNTPOFF:
      output_addr_const (file, op);
      if (TARGET_64BIT)
	fputs (ASSEMBLER_DIALECT == ASM_ATT ?
	       "@gottpoff(%rip)" : "@gottpoff[rip]", file);
      else
	fputs ("@gotntpoff", file);
      break;
    case UNSPEC_INDNTPOFF:
      output_addr_const (file, op);
      fputs ("@indntpoff", file);
      break;
#if TARGET_MACHO
    case UNSPEC_MACHOPIC_OFFSET:
      output_addr_const (file, op);
      putc ('-', file);
      machopic_output_function_base_name (file);
      break;
#endif

    case UNSPEC_STACK_CHECK:
      {
	int offset;

	gcc_assert (flag_split_stack);

#ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
	offset = TARGET_THREAD_SPLIT_STACK_OFFSET;
#else
	gcc_unreachable ();
#endif

	fprintf (file, "%s:%d", TARGET_64BIT ? "%fs" : "%gs", offset);
      }
      break;

    default:
      return false;
    }

  return true;
}

/* Split one or more double-mode RTL references into pairs of half-mode
   references.  The RTL can be REG, offsettable MEM, integer constant, or
   CONST_DOUBLE.  "operands" is a pointer to an array of double-mode RTLs to
   split and "num" is its length.  lo_half and hi_half are output arrays
   that parallel "operands".  */

void
split_double_mode (machine_mode mode, rtx operands[],
		   int num, rtx lo_half[], rtx hi_half[])
{
  machine_mode half_mode;
  unsigned int byte;

  switch (mode)
    {
    case TImode:
      half_mode = DImode;
      break;
    case DImode:
      half_mode = SImode;
      break;
    default:
      gcc_unreachable ();
    }

  byte = GET_MODE_SIZE (half_mode);

  while (num--)
    {
      rtx op = operands[num];

      /* simplify_subreg refuse to split volatile memory addresses,
         but we still have to handle it.  */
      if (MEM_P (op))
	{
	  lo_half[num] = adjust_address (op, half_mode, 0);
	  hi_half[num] = adjust_address (op, half_mode, byte);
	}
      else
	{
	  lo_half[num] = simplify_gen_subreg (half_mode, op,
					      GET_MODE (op) == VOIDmode
					      ? mode : GET_MODE (op), 0);
	  hi_half[num] = simplify_gen_subreg (half_mode, op,
					      GET_MODE (op) == VOIDmode
					      ? mode : GET_MODE (op), byte);
	}
    }
}

/* Output code to perform a 387 binary operation in INSN, one of PLUS,
   MINUS, MULT or DIV.  OPERANDS are the insn operands, where operands[3]
   is the expression of the binary operation.  The output may either be
   emitted here, or returned to the caller, like all output_* functions.

   There is no guarantee that the operands are the same mode, as they
   might be within FLOAT or FLOAT_EXTEND expressions.  */

#ifndef SYSV386_COMPAT
/* Set to 1 for compatibility with brain-damaged assemblers.  No-one
   wants to fix the assemblers because that causes incompatibility
   with gcc.  No-one wants to fix gcc because that causes
   incompatibility with assemblers...  You can use the option of
   -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way.  */
#define SYSV386_COMPAT 1
#endif

const char *
output_387_binary_op (rtx_insn *insn, rtx *operands)
{
  static char buf[40];
  const char *p;
  const char *ssep;
  int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]);

  /* Even if we do not want to check the inputs, this documents input
     constraints.  Which helps in understanding the following code.  */
  if (flag_checking)
    {
      if (STACK_REG_P (operands[0])
	  && ((REG_P (operands[1])
	       && REGNO (operands[0]) == REGNO (operands[1])
	       && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
	      || (REG_P (operands[2])
		  && REGNO (operands[0]) == REGNO (operands[2])
		  && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
	  && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
	; /* ok */
      else
	gcc_assert (is_sse);
    }

  switch (GET_CODE (operands[3]))
    {
    case PLUS:
      if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
	  || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
	p = "fiadd";
      else
	p = "fadd";
      ssep = "vadd";
      break;

    case MINUS:
      if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
	  || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
	p = "fisub";
      else
	p = "fsub";
      ssep = "vsub";
      break;

    case MULT:
      if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
	  || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
	p = "fimul";
      else
	p = "fmul";
      ssep = "vmul";
      break;

    case DIV:
      if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
	  || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
	p = "fidiv";
      else
	p = "fdiv";
      ssep = "vdiv";
      break;

    default:
      gcc_unreachable ();
    }

  if (is_sse)
   {
     if (TARGET_AVX)
       {
	 strcpy (buf, ssep);
	 if (GET_MODE (operands[0]) == SFmode)
	   strcat (buf, "ss\t{%2, %1, %0|%0, %1, %2}");
	 else
	   strcat (buf, "sd\t{%2, %1, %0|%0, %1, %2}");
       }
     else
       {
	 strcpy (buf, ssep + 1);
	 if (GET_MODE (operands[0]) == SFmode)
	   strcat (buf, "ss\t{%2, %0|%0, %2}");
	 else
	   strcat (buf, "sd\t{%2, %0|%0, %2}");
       }
      return buf;
   }
  strcpy (buf, p);

  switch (GET_CODE (operands[3]))
    {
    case MULT:
    case PLUS:
      if (REG_P (operands[2]) && REGNO (operands[0]) == REGNO (operands[2]))
	std::swap (operands[1], operands[2]);

      /* know operands[0] == operands[1].  */

      if (MEM_P (operands[2]))
	{
	  p = "%Z2\t%2";
	  break;
	}

      if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
	{
	  if (STACK_TOP_P (operands[0]))
	    /* How is it that we are storing to a dead operand[2]?
	       Well, presumably operands[1] is dead too.  We can't
	       store the result to st(0) as st(0) gets popped on this
	       instruction.  Instead store to operands[2] (which I
	       think has to be st(1)).  st(1) will be popped later.
	       gcc <= 2.8.1 didn't have this check and generated
	       assembly code that the Unixware assembler rejected.  */
	    p = "p\t{%0, %2|%2, %0}";	/* st(1) = st(0) op st(1); pop */
	  else
	    p = "p\t{%2, %0|%0, %2}";	/* st(r1) = st(r1) op st(0); pop */
	  break;
	}

      if (STACK_TOP_P (operands[0]))
	p = "\t{%y2, %0|%0, %y2}";	/* st(0) = st(0) op st(r2) */
      else
	p = "\t{%2, %0|%0, %2}";	/* st(r1) = st(r1) op st(0) */
      break;

    case MINUS:
    case DIV:
      if (MEM_P (operands[1]))
	{
	  p = "r%Z1\t%1";
	  break;
	}

      if (MEM_P (operands[2]))
	{
	  p = "%Z2\t%2";
	  break;
	}

      if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
	{
#if SYSV386_COMPAT
	  /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
	     derived assemblers, confusingly reverse the direction of
	     the operation for fsub{r} and fdiv{r} when the
	     destination register is not st(0).  The Intel assembler
	     doesn't have this brain damage.  Read !SYSV386_COMPAT to
	     figure out what the hardware really does.  */
	  if (STACK_TOP_P (operands[0]))
	    p = "{p\t%0, %2|rp\t%2, %0}";
	  else
	    p = "{rp\t%2, %0|p\t%0, %2}";
#else
	  if (STACK_TOP_P (operands[0]))
	    /* As above for fmul/fadd, we can't store to st(0).  */
	    p = "rp\t{%0, %2|%2, %0}";	/* st(1) = st(0) op st(1); pop */
	  else
	    p = "p\t{%2, %0|%0, %2}";	/* st(r1) = st(r1) op st(0); pop */
#endif
	  break;
	}

      if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
	{
#if SYSV386_COMPAT
	  if (STACK_TOP_P (operands[0]))
	    p = "{rp\t%0, %1|p\t%1, %0}";
	  else
	    p = "{p\t%1, %0|rp\t%0, %1}";
#else
	  if (STACK_TOP_P (operands[0]))
	    p = "p\t{%0, %1|%1, %0}";	/* st(1) = st(1) op st(0); pop */
	  else
	    p = "rp\t{%1, %0|%0, %1}";	/* st(r2) = st(0) op st(r2); pop */
#endif
	  break;
	}

      if (STACK_TOP_P (operands[0]))
	{
	  if (STACK_TOP_P (operands[1]))
	    p = "\t{%y2, %0|%0, %y2}";	/* st(0) = st(0) op st(r2) */
	  else
	    p = "r\t{%y1, %0|%0, %y1}";	/* st(0) = st(r1) op st(0) */
	  break;
	}
      else if (STACK_TOP_P (operands[1]))
	{
#if SYSV386_COMPAT
	  p = "{\t%1, %0|r\t%0, %1}";
#else
	  p = "r\t{%1, %0|%0, %1}";	/* st(r2) = st(0) op st(r2) */
#endif
	}
      else
	{
#if SYSV386_COMPAT
	  p = "{r\t%2, %0|\t%0, %2}";
#else
	  p = "\t{%2, %0|%0, %2}";	/* st(r1) = st(r1) op st(0) */
#endif
	}
      break;

    default:
      gcc_unreachable ();
    }

  strcat (buf, p);
  return buf;
}

/* Return needed mode for entity in optimize_mode_switching pass.  */

static int
ix86_dirflag_mode_needed (rtx_insn *insn)
{
  if (CALL_P (insn))
    {
      if (cfun->machine->func_type == TYPE_NORMAL)
	return X86_DIRFLAG_ANY;
      else
	/* No need to emit CLD in interrupt handler for TARGET_CLD.  */
	return TARGET_CLD ? X86_DIRFLAG_ANY : X86_DIRFLAG_RESET;
    }

  if (recog_memoized (insn) < 0)
    return X86_DIRFLAG_ANY;

  if (get_attr_type (insn) == TYPE_STR)
    {
      /* Emit cld instruction if stringops are used in the function.  */
      if (cfun->machine->func_type == TYPE_NORMAL)
	return TARGET_CLD ? X86_DIRFLAG_RESET : X86_DIRFLAG_ANY;
      else
	return X86_DIRFLAG_RESET;
    }

  return X86_DIRFLAG_ANY;
}

/* Check if a 256bit AVX register is referenced inside of EXP.   */

static bool
ix86_check_avx256_register (const_rtx exp)
{
  if (SUBREG_P (exp))
    exp = SUBREG_REG (exp);

  return (REG_P (exp)
	  && VALID_AVX256_REG_OR_OI_MODE (GET_MODE (exp)));
}

/* Return needed mode for entity in optimize_mode_switching pass.  */

static int
ix86_avx_u128_mode_needed (rtx_insn *insn)
{
  if (CALL_P (insn))
    {
      rtx link;

      /* Needed mode is set to AVX_U128_CLEAN if there are
	 no 256bit modes used in function arguments.  */
      for (link = CALL_INSN_FUNCTION_USAGE (insn);
	   link;
	   link = XEXP (link, 1))
	{
	  if (GET_CODE (XEXP (link, 0)) == USE)
	    {
	      rtx arg = XEXP (XEXP (link, 0), 0);

	      if (ix86_check_avx256_register (arg))
		return AVX_U128_DIRTY;
	    }
	}

      return AVX_U128_CLEAN;
    }

  /* Require DIRTY mode if a 256bit AVX register is referenced.  Hardware
     changes state only when a 256bit register is written to, but we need
     to prevent the compiler from moving optimal insertion point above
     eventual read from 256bit register.  */
  subrtx_iterator::array_type array;
  FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
    if (ix86_check_avx256_register (*iter))
      return AVX_U128_DIRTY;

  return AVX_U128_ANY;
}

/* Return mode that i387 must be switched into
   prior to the execution of insn.  */

static int
ix86_i387_mode_needed (int entity, rtx_insn *insn)
{
  enum attr_i387_cw mode;

  /* The mode UNINITIALIZED is used to store control word after a
     function call or ASM pattern.  The mode ANY specify that function
     has no requirements on the control word and make no changes in the
     bits we are interested in.  */

  if (CALL_P (insn)
      || (NONJUMP_INSN_P (insn)
	  && (asm_noperands (PATTERN (insn)) >= 0
	      || GET_CODE (PATTERN (insn)) == ASM_INPUT)))
    return I387_CW_UNINITIALIZED;

  if (recog_memoized (insn) < 0)
    return I387_CW_ANY;

  mode = get_attr_i387_cw (insn);

  switch (entity)
    {
    case I387_TRUNC:
      if (mode == I387_CW_TRUNC)
	return mode;
      break;

    case I387_FLOOR:
      if (mode == I387_CW_FLOOR)
	return mode;
      break;

    case I387_CEIL:
      if (mode == I387_CW_CEIL)
	return mode;
      break;

    case I387_MASK_PM:
      if (mode == I387_CW_MASK_PM)
	return mode;
      break;

    default:
      gcc_unreachable ();
    }

  return I387_CW_ANY;
}

/* Return mode that entity must be switched into
   prior to the execution of insn.  */

static int
ix86_mode_needed (int entity, rtx_insn *insn)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      return ix86_dirflag_mode_needed (insn);
    case AVX_U128:
      return ix86_avx_u128_mode_needed (insn);
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
    case I387_MASK_PM:
      return ix86_i387_mode_needed (entity, insn);
    default:
      gcc_unreachable ();
    }
  return 0;
}

/* Check if a 256bit AVX register is referenced in stores.   */
 
static void
ix86_check_avx256_stores (rtx dest, const_rtx, void *data)
 {
   if (ix86_check_avx256_register (dest))
    {
      bool *used = (bool *) data;
      *used = true;
    }
 } 

/* Calculate mode of upper 128bit AVX registers after the insn.  */

static int
ix86_avx_u128_mode_after (int mode, rtx_insn *insn)
{
  rtx pat = PATTERN (insn);

  if (vzeroupper_operation (pat, VOIDmode)
      || vzeroall_operation (pat, VOIDmode))
    return AVX_U128_CLEAN;

  /* We know that state is clean after CALL insn if there are no
     256bit registers used in the function return register.  */
  if (CALL_P (insn))
    {
      bool avx_reg256_found = false;
      note_stores (pat, ix86_check_avx256_stores, &avx_reg256_found);

      return avx_reg256_found ? AVX_U128_DIRTY : AVX_U128_CLEAN;
    }

  /* Otherwise, return current mode.  Remember that if insn
     references AVX 256bit registers, the mode was already changed
     to DIRTY from MODE_NEEDED.  */
  return mode;
}

/* Return the mode that an insn results in.  */

static int
ix86_mode_after (int entity, int mode, rtx_insn *insn)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      return mode;
    case AVX_U128:
      return ix86_avx_u128_mode_after (mode, insn);
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
    case I387_MASK_PM:
      return mode;
    default:
      gcc_unreachable ();
    }
}

static int
ix86_dirflag_mode_entry (void)
{
  /* For TARGET_CLD or in the interrupt handler we can't assume
     direction flag state at function entry.  */
  if (TARGET_CLD
      || cfun->machine->func_type != TYPE_NORMAL)
    return X86_DIRFLAG_ANY;

  return X86_DIRFLAG_RESET;
}

static int
ix86_avx_u128_mode_entry (void)
{
  tree arg;

  /* Entry mode is set to AVX_U128_DIRTY if there are
     256bit modes used in function arguments.  */
  for (arg = DECL_ARGUMENTS (current_function_decl); arg;
       arg = TREE_CHAIN (arg))
    {
      rtx incoming = DECL_INCOMING_RTL (arg);

      if (incoming && ix86_check_avx256_register (incoming))
	return AVX_U128_DIRTY;
    }

  return AVX_U128_CLEAN;
}

/* Return a mode that ENTITY is assumed to be
   switched to at function entry.  */

static int
ix86_mode_entry (int entity)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      return ix86_dirflag_mode_entry ();
    case AVX_U128:
      return ix86_avx_u128_mode_entry ();
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
    case I387_MASK_PM:
      return I387_CW_ANY;
    default:
      gcc_unreachable ();
    }
}

static int
ix86_avx_u128_mode_exit (void)
{
  rtx reg = crtl->return_rtx;

  /* Exit mode is set to AVX_U128_DIRTY if there are
     256bit modes used in the function return register.  */
  if (reg && ix86_check_avx256_register (reg))
    return AVX_U128_DIRTY;

  return AVX_U128_CLEAN;
}

/* Return a mode that ENTITY is assumed to be
   switched to at function exit.  */

static int
ix86_mode_exit (int entity)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      return X86_DIRFLAG_ANY;
    case AVX_U128:
      return ix86_avx_u128_mode_exit ();
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
    case I387_MASK_PM:
      return I387_CW_ANY;
    default:
      gcc_unreachable ();
    }
}

static int
ix86_mode_priority (int, int n)
{
  return n;
}

/* Output code to initialize control word copies used by trunc?f?i and
   rounding patterns.  CURRENT_MODE is set to current control word,
   while NEW_MODE is set to new control word.  */

static void
emit_i387_cw_initialization (int mode)
{
  rtx stored_mode = assign_386_stack_local (HImode, SLOT_CW_STORED);
  rtx new_mode;

  enum ix86_stack_slot slot;

  rtx reg = gen_reg_rtx (HImode);

  emit_insn (gen_x86_fnstcw_1 (stored_mode));
  emit_move_insn (reg, copy_rtx (stored_mode));

  if (TARGET_64BIT || TARGET_PARTIAL_REG_STALL
      || optimize_insn_for_size_p ())
    {
      switch (mode)
	{
	case I387_CW_TRUNC:
	  /* round toward zero (truncate) */
	  emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00)));
	  slot = SLOT_CW_TRUNC;
	  break;

	case I387_CW_FLOOR:
	  /* round down toward -oo */
	  emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
	  emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400)));
	  slot = SLOT_CW_FLOOR;
	  break;

	case I387_CW_CEIL:
	  /* round up toward +oo */
	  emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
	  emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800)));
	  slot = SLOT_CW_CEIL;
	  break;

	case I387_CW_MASK_PM:
	  /* mask precision exception for nearbyint() */
	  emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
	  slot = SLOT_CW_MASK_PM;
	  break;

	default:
	  gcc_unreachable ();
	}
    }
  else
    {
      switch (mode)
	{
	case I387_CW_TRUNC:
	  /* round toward zero (truncate) */
	  emit_insn (gen_insvsi_1 (reg, GEN_INT (0xc)));
	  slot = SLOT_CW_TRUNC;
	  break;

	case I387_CW_FLOOR:
	  /* round down toward -oo */
	  emit_insn (gen_insvsi_1 (reg, GEN_INT (0x4)));
	  slot = SLOT_CW_FLOOR;
	  break;

	case I387_CW_CEIL:
	  /* round up toward +oo */
	  emit_insn (gen_insvsi_1 (reg, GEN_INT (0x8)));
	  slot = SLOT_CW_CEIL;
	  break;

	case I387_CW_MASK_PM:
	  /* mask precision exception for nearbyint() */
	  emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
	  slot = SLOT_CW_MASK_PM;
	  break;

	default:
	  gcc_unreachable ();
	}
    }

  gcc_assert (slot < MAX_386_STACK_LOCALS);

  new_mode = assign_386_stack_local (HImode, slot);
  emit_move_insn (new_mode, reg);
}

/* Emit vzeroupper.  */

void
ix86_avx_emit_vzeroupper (HARD_REG_SET regs_live)
{
  int i;

  /* Cancel automatic vzeroupper insertion if there are
     live call-saved SSE registers at the insertion point.  */

  for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
    if (TEST_HARD_REG_BIT (regs_live, i) && !call_used_regs[i])
      return;

  if (TARGET_64BIT)
    for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
      if (TEST_HARD_REG_BIT (regs_live, i) && !call_used_regs[i])
	return;

  emit_insn (gen_avx_vzeroupper ());
}

/* Generate one or more insns to set ENTITY to MODE.  */

/* Generate one or more insns to set ENTITY to MODE.  HARD_REG_LIVE
   is the set of hard registers live at the point where the insn(s)
   are to be inserted.  */

static void
ix86_emit_mode_set (int entity, int mode, int prev_mode ATTRIBUTE_UNUSED,
		    HARD_REG_SET regs_live)
{
  switch (entity)
    {
    case X86_DIRFLAG:
      if (mode == X86_DIRFLAG_RESET)
	emit_insn (gen_cld ());
      break;
    case AVX_U128:
      if (mode == AVX_U128_CLEAN)
	ix86_avx_emit_vzeroupper (regs_live);
      break;
    case I387_TRUNC:
    case I387_FLOOR:
    case I387_CEIL:
    case I387_MASK_PM:
      if (mode != I387_CW_ANY
	  && mode != I387_CW_UNINITIALIZED)
	emit_i387_cw_initialization (mode);
      break;
    default:
      gcc_unreachable ();
    }
}

/* Output code for INSN to convert a float to a signed int.  OPERANDS
   are the insn operands.  The output may be [HSD]Imode and the input
   operand may be [SDX]Fmode.  */

const char *
output_fix_trunc (rtx_insn *insn, rtx *operands, bool fisttp)
{
  int stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
  int dimode_p = GET_MODE (operands[0]) == DImode;
  int round_mode = get_attr_i387_cw (insn);

  /* Jump through a hoop or two for DImode, since the hardware has no
     non-popping instruction.  We used to do this a different way, but
     that was somewhat fragile and broke with post-reload splitters.  */
  if ((dimode_p || fisttp) && !stack_top_dies)
    output_asm_insn ("fld\t%y1", operands);

  gcc_assert (STACK_TOP_P (operands[1]));
  gcc_assert (MEM_P (operands[0]));
  gcc_assert (GET_MODE (operands[1]) != TFmode);

  if (fisttp)
      output_asm_insn ("fisttp%Z0\t%0", operands);
  else
    {
      if (round_mode != I387_CW_ANY)
	output_asm_insn ("fldcw\t%3", operands);
      if (stack_top_dies || dimode_p)
	output_asm_insn ("fistp%Z0\t%0", operands);
      else
	output_asm_insn ("fist%Z0\t%0", operands);
      if (round_mode != I387_CW_ANY)
	output_asm_insn ("fldcw\t%2", operands);
    }

  return "";
}

/* Output code for x87 ffreep insn.  The OPNO argument, which may only
   have the values zero or one, indicates the ffreep insn's operand
   from the OPERANDS array.  */

static const char *
output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
{
  if (TARGET_USE_FFREEP)
#ifdef HAVE_AS_IX86_FFREEP
    return opno ? "ffreep\t%y1" : "ffreep\t%y0";
#else
    {
      static char retval[32];
      int regno = REGNO (operands[opno]);

      gcc_assert (STACK_REGNO_P (regno));

      regno -= FIRST_STACK_REG;

      snprintf (retval, sizeof (retval), ASM_SHORT "0xc%ddf", regno);
      return retval;
    }
#endif

  return opno ? "fstp\t%y1" : "fstp\t%y0";
}


/* Output code for INSN to compare OPERANDS.  EFLAGS_P is 1 when fcomi
   should be used.  UNORDERED_P is true when fucom should be used.  */

const char *
output_fp_compare (rtx_insn *insn, rtx *operands, bool eflags_p, bool unordered_p)
{
  int stack_top_dies;
  rtx cmp_op0, cmp_op1;
  int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]);

  if (eflags_p)
    {
      cmp_op0 = operands[0];
      cmp_op1 = operands[1];
    }
  else
    {
      cmp_op0 = operands[1];
      cmp_op1 = operands[2];
    }

  if (is_sse)
    {
      if (GET_MODE (operands[0]) == SFmode)
	if (unordered_p)
	  return "%vucomiss\t{%1, %0|%0, %1}";
	else
	  return "%vcomiss\t{%1, %0|%0, %1}";
      else
	if (unordered_p)
	  return "%vucomisd\t{%1, %0|%0, %1}";
	else
	  return "%vcomisd\t{%1, %0|%0, %1}";
    }

  gcc_assert (STACK_TOP_P (cmp_op0));

  stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;

  if (cmp_op1 == CONST0_RTX (GET_MODE (cmp_op1)))
    {
      if (stack_top_dies)
	{
	  output_asm_insn ("ftst\n\tfnstsw\t%0", operands);
	  return output_387_ffreep (operands, 1);
	}
      else
	return "ftst\n\tfnstsw\t%0";
    }

  if (STACK_REG_P (cmp_op1)
      && stack_top_dies
      && find_regno_note (insn, REG_DEAD, REGNO (cmp_op1))
      && REGNO (cmp_op1) != FIRST_STACK_REG)
    {
      /* If both the top of the 387 stack dies, and the other operand
	 is also a stack register that dies, then this must be a
	 `fcompp' float compare */

      if (eflags_p)
	{
	  /* There is no double popping fcomi variant.  Fortunately,
	     eflags is immune from the fstp's cc clobbering.  */
	  if (unordered_p)
	    output_asm_insn ("fucomip\t{%y1, %0|%0, %y1}", operands);
	  else
	    output_asm_insn ("fcomip\t{%y1, %0|%0, %y1}", operands);
	  return output_387_ffreep (operands, 0);
	}
      else
	{
	  if (unordered_p)
	    return "fucompp\n\tfnstsw\t%0";
	  else
	    return "fcompp\n\tfnstsw\t%0";
	}
    }
  else
    {
      /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies.  */

      static const char * const alt[16] =
      {
	"fcom%Z2\t%y2\n\tfnstsw\t%0",
	"fcomp%Z2\t%y2\n\tfnstsw\t%0",
	"fucom%Z2\t%y2\n\tfnstsw\t%0",
	"fucomp%Z2\t%y2\n\tfnstsw\t%0",

	"ficom%Z2\t%y2\n\tfnstsw\t%0",
	"ficomp%Z2\t%y2\n\tfnstsw\t%0",
	NULL,
	NULL,

	"fcomi\t{%y1, %0|%0, %y1}",
	"fcomip\t{%y1, %0|%0, %y1}",
	"fucomi\t{%y1, %0|%0, %y1}",
	"fucomip\t{%y1, %0|%0, %y1}",

	NULL,
	NULL,
	NULL,
	NULL
      };

      int mask;
      const char *ret;

      mask  = eflags_p << 3;
      mask |= (GET_MODE_CLASS (GET_MODE (cmp_op1)) == MODE_INT) << 2;
      mask |= unordered_p << 1;
      mask |= stack_top_dies;

      gcc_assert (mask < 16);
      ret = alt[mask];
      gcc_assert (ret);

      return ret;
    }
}

void
ix86_output_addr_vec_elt (FILE *file, int value)
{
  const char *directive = ASM_LONG;

#ifdef ASM_QUAD
  if (TARGET_LP64)
    directive = ASM_QUAD;
#else
  gcc_assert (!TARGET_64BIT);
#endif

  fprintf (file, "%s%s%d\n", directive, LPREFIX, value);
}

void
ix86_output_addr_diff_elt (FILE *file, int value, int rel)
{
  const char *directive = ASM_LONG;

#ifdef ASM_QUAD
  if (TARGET_64BIT && CASE_VECTOR_MODE == DImode)
    directive = ASM_QUAD;
#else
  gcc_assert (!TARGET_64BIT);
#endif
  /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand.  */
  if (TARGET_64BIT || TARGET_VXWORKS_RTP)
    fprintf (file, "%s%s%d-%s%d\n",
	     directive, LPREFIX, value, LPREFIX, rel);
  else if (HAVE_AS_GOTOFF_IN_DATA)
    fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value);
#if TARGET_MACHO
  else if (TARGET_MACHO)
    {
      fprintf (file, ASM_LONG "%s%d-", LPREFIX, value);
      machopic_output_function_base_name (file);
      putc ('\n', file);
    }
#endif
  else
    asm_fprintf (file, ASM_LONG "%U%s+[.-%s%d]\n",
		 GOT_SYMBOL_NAME, LPREFIX, value);
}

/* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
   for the target.  */

void
ix86_expand_clear (rtx dest)
{
  rtx tmp;

  /* We play register width games, which are only valid after reload.  */
  gcc_assert (reload_completed);

  /* Avoid HImode and its attendant prefix byte.  */
  if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
    dest = gen_rtx_REG (SImode, REGNO (dest));
  tmp = gen_rtx_SET (dest, const0_rtx);

  if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
    {
      rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
      tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
    }

  emit_insn (tmp);
}

/* X is an unchanging MEM.  If it is a constant pool reference, return
   the constant pool rtx, else NULL.  */

rtx
maybe_get_pool_constant (rtx x)
{
  x = ix86_delegitimize_address (XEXP (x, 0));

  if (GET_CODE (x) == SYMBOL_REF && CONSTANT_POOL_ADDRESS_P (x))
    return get_pool_constant (x);

  return NULL_RTX;
}

void
ix86_expand_move (machine_mode mode, rtx operands[])
{
  rtx op0, op1;
  rtx tmp, addend = NULL_RTX;
  enum tls_model model;

  op0 = operands[0];
  op1 = operands[1];

  switch (GET_CODE (op1))
    {
    case CONST:
      tmp = XEXP (op1, 0);

      if (GET_CODE (tmp) != PLUS
	  || GET_CODE (XEXP (tmp, 0)) != SYMBOL_REF)
	break;

      op1 = XEXP (tmp, 0);
      addend = XEXP (tmp, 1);
      /* FALLTHRU */

    case SYMBOL_REF:
      model = SYMBOL_REF_TLS_MODEL (op1);

      if (model)
	op1 = legitimize_tls_address (op1, model, true);
      else if (ix86_force_load_from_GOT_p (op1))
	{
	  /* Load the external function address via GOT slot to avoid PLT.  */
	  op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op1),
				(TARGET_64BIT
				 ? UNSPEC_GOTPCREL
				 : UNSPEC_GOT));
	  op1 = gen_rtx_CONST (Pmode, op1);
	  op1 = gen_const_mem (Pmode, op1);
	  set_mem_alias_set (op1, ix86_GOT_alias_set ());
	}
      else
	{
	  tmp = legitimize_pe_coff_symbol (op1, addend != NULL_RTX);
	  if (tmp)
	    {
	      op1 = tmp;
	      if (!addend)
		break;
	    }
	  else
	    {
	      op1 = operands[1];
	      break;
	    }
	}

      if (addend)
	{
	  op1 = force_operand (op1, NULL_RTX);
	  op1 = expand_simple_binop (Pmode, PLUS, op1, addend,
				     op0, 1, OPTAB_DIRECT);
	}
      else
	op1 = force_operand (op1, op0);

      if (op1 == op0)
	return;

      op1 = convert_to_mode (mode, op1, 1);

    default:
      break;
    }

  if ((flag_pic || MACHOPIC_INDIRECT)
      && symbolic_operand (op1, mode))
    {
      if (TARGET_MACHO && !TARGET_64BIT)
	{
#if TARGET_MACHO
	  /* dynamic-no-pic */
	  if (MACHOPIC_INDIRECT)
	    {
	      rtx temp = (op0 && REG_P (op0) && mode == Pmode)
			 ? op0 : gen_reg_rtx (Pmode);
	      op1 = machopic_indirect_data_reference (op1, temp);
	      if (MACHOPIC_PURE)
		op1 = machopic_legitimize_pic_address (op1, mode,
						       temp == op1 ? 0 : temp);
	    }
	  if (op0 != op1 && GET_CODE (op0) != MEM)
	    {
	      rtx insn = gen_rtx_SET (op0, op1);
	      emit_insn (insn);
	      return;
	    }
	  if (GET_CODE (op0) == MEM)
	    op1 = force_reg (Pmode, op1);
	  else
	    {
	      rtx temp = op0;
	      if (GET_CODE (temp) != REG)
		temp = gen_reg_rtx (Pmode);
	      temp = legitimize_pic_address (op1, temp);
	      if (temp == op0)
	    return;
	      op1 = temp;
	    }
      /* dynamic-no-pic */
#endif
	}
      else
	{
	  if (MEM_P (op0))
	    op1 = force_reg (mode, op1);
	  else if (!(TARGET_64BIT && x86_64_movabs_operand (op1, DImode)))
	    {
	      rtx reg = can_create_pseudo_p () ? NULL_RTX : op0;
	      op1 = legitimize_pic_address (op1, reg);
	      if (op0 == op1)
		return;
	      op1 = convert_to_mode (mode, op1, 1);
	    }
	}
    }
  else
    {
      if (MEM_P (op0)
	  && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
	      || !push_operand (op0, mode))
	  && MEM_P (op1))
	op1 = force_reg (mode, op1);

      if (push_operand (op0, mode)
	  && ! general_no_elim_operand (op1, mode))
	op1 = copy_to_mode_reg (mode, op1);

      /* Force large constants in 64bit compilation into register
	 to get them CSEed.  */
      if (can_create_pseudo_p ()
	  && (mode == DImode) && TARGET_64BIT
	  && immediate_operand (op1, mode)
	  && !x86_64_zext_immediate_operand (op1, VOIDmode)
	  && !register_operand (op0, mode)
	  && optimize)
	op1 = copy_to_mode_reg (mode, op1);

      if (can_create_pseudo_p ()
	  && CONST_DOUBLE_P (op1))
	{
	  /* If we are loading a floating point constant to a register,
	     force the value to memory now, since we'll get better code
	     out the back end.  */

	  op1 = validize_mem (force_const_mem (mode, op1));
	  if (!register_operand (op0, mode))
	    {
	      rtx temp = gen_reg_rtx (mode);
	      emit_insn (gen_rtx_SET (temp, op1));
	      emit_move_insn (op0, temp);
	      return;
	    }
	}
    }

  emit_insn (gen_rtx_SET (op0, op1));
}

void
ix86_expand_vector_move (machine_mode mode, rtx operands[])
{
  rtx op0 = operands[0], op1 = operands[1];
  /* Use GET_MODE_BITSIZE instead of GET_MODE_ALIGNMENT for IA MCU
     psABI since the biggest alignment is 4 byte for IA MCU psABI.  */
  unsigned int align = (TARGET_IAMCU
			? GET_MODE_BITSIZE (mode)
			: GET_MODE_ALIGNMENT (mode));

  if (push_operand (op0, VOIDmode))
    op0 = emit_move_resolve_push (mode, op0);

  /* Force constants other than zero into memory.  We do not know how
     the instructions used to build constants modify the upper 64 bits
     of the register, once we have that information we may be able
     to handle some of them more efficiently.  */
  if (can_create_pseudo_p ()
      && (CONSTANT_P (op1)
	  || (SUBREG_P (op1)
	      && CONSTANT_P (SUBREG_REG (op1))))
      && ((register_operand (op0, mode)
	   && !standard_sse_constant_p (op1, mode))
	  /* ix86_expand_vector_move_misalign() does not like constants.  */
	  || (SSE_REG_MODE_P (mode)
	      && MEM_P (op0)
	      && MEM_ALIGN (op0) < align)))
    {
      if (SUBREG_P (op1))
	{
	  machine_mode imode = GET_MODE (SUBREG_REG (op1));
	  rtx r = force_const_mem (imode, SUBREG_REG (op1));
	  if (r)
	    r = validize_mem (r);
	  else
	    r = force_reg (imode, SUBREG_REG (op1));
	  op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
	}
      else
	op1 = validize_mem (force_const_mem (mode, op1));
    }

  /* We need to check memory alignment for SSE mode since attribute
     can make operands unaligned.  */
  if (can_create_pseudo_p ()
      && SSE_REG_MODE_P (mode)
      && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
	  || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
    {
      rtx tmp[2];

      /* ix86_expand_vector_move_misalign() does not like both
	 arguments in memory.  */
      if (!register_operand (op0, mode)
	  && !register_operand (op1, mode))
	op1 = force_reg (mode, op1);

      tmp[0] = op0; tmp[1] = op1;
      ix86_expand_vector_move_misalign (mode, tmp);
      return;
    }

  /* Make operand1 a register if it isn't already.  */
  if (can_create_pseudo_p ()
      && !register_operand (op0, mode)
      && !register_operand (op1, mode))
    {
      emit_move_insn (op0, force_reg (GET_MODE (op0), op1));
      return;
    }

  emit_insn (gen_rtx_SET (op0, op1));
}

/* Split 32-byte AVX unaligned load and store if needed.  */

static void
ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
{
  rtx m;
  rtx (*extract) (rtx, rtx, rtx);
  machine_mode mode;

  if ((MEM_P (op1) && !TARGET_AVX256_SPLIT_UNALIGNED_LOAD)
      || (MEM_P (op0) && !TARGET_AVX256_SPLIT_UNALIGNED_STORE))
    {
      emit_insn (gen_rtx_SET (op0, op1));
      return;
    }

  rtx orig_op0 = NULL_RTX;
  mode = GET_MODE (op0);
  switch (GET_MODE_CLASS (mode))
    {
    case MODE_VECTOR_INT:
    case MODE_INT:
      if (mode != V32QImode)
	{
	  if (!MEM_P (op0))
	    {
	      orig_op0 = op0;
	      op0 = gen_reg_rtx (V32QImode);
	    }
	  else
	    op0 = gen_lowpart (V32QImode, op0);
	  op1 = gen_lowpart (V32QImode, op1);
	  mode = V32QImode;
	}
      break;
    case MODE_VECTOR_FLOAT:
      break;
    default:
      gcc_unreachable ();
    }

  switch (mode)
    {
    default:
      gcc_unreachable ();
    case V32QImode:
      extract = gen_avx_vextractf128v32qi;
      mode = V16QImode;
      break;
    case V8SFmode:
      extract = gen_avx_vextractf128v8sf;
      mode = V4SFmode;
      break;
    case V4DFmode:
      extract = gen_avx_vextractf128v4df;
      mode = V2DFmode;
      break;
    }

  if (MEM_P (op1))
    {
      rtx r = gen_reg_rtx (mode);
      m = adjust_address (op1, mode, 0);
      emit_move_insn (r, m);
      m = adjust_address (op1, mode, 16);
      r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
      emit_move_insn (op0, r);
    }
  else if (MEM_P (op0))
    {
      m = adjust_address (op0, mode, 0);
      emit_insn (extract (m, op1, const0_rtx));
      m = adjust_address (op0, mode, 16);
      emit_insn (extract (m, copy_rtx (op1), const1_rtx));
    }
  else
    gcc_unreachable ();

  if (orig_op0)
    emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
}

/* Implement the movmisalign patterns for SSE.  Non-SSE modes go
   straight to ix86_expand_vector_move.  */
/* Code generation for scalar reg-reg moves of single and double precision data:
     if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
       movaps reg, reg
     else
       movss reg, reg
     if (x86_sse_partial_reg_dependency == true)
       movapd reg, reg
     else
       movsd reg, reg

   Code generation for scalar loads of double precision data:
     if (x86_sse_split_regs == true)
       movlpd mem, reg      (gas syntax)
     else
       movsd mem, reg

   Code generation for unaligned packed loads of single precision data
   (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
     if (x86_sse_unaligned_move_optimal)
       movups mem, reg

     if (x86_sse_partial_reg_dependency == true)
       {
         xorps  reg, reg
         movlps mem, reg
         movhps mem+8, reg
       }
     else
       {
         movlps mem, reg
         movhps mem+8, reg
       }

   Code generation for unaligned packed loads of double precision data
   (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
     if (x86_sse_unaligned_move_optimal)
       movupd mem, reg

     if (x86_sse_split_regs == true)
       {
         movlpd mem, reg
         movhpd mem+8, reg
       }
     else
       {
         movsd  mem, reg
         movhpd mem+8, reg
       }
 */

void
ix86_expand_vector_move_misalign (machine_mode mode, rtx operands[])
{
  rtx op0, op1, m;

  op0 = operands[0];
  op1 = operands[1];

  /* Use unaligned load/store for AVX512 or when optimizing for size.  */
  if (GET_MODE_SIZE (mode) == 64 || optimize_insn_for_size_p ())
    {
      emit_insn (gen_rtx_SET (op0, op1));
      return;
    }

  if (TARGET_AVX)
    {
      if (GET_MODE_SIZE (mode) == 32)
	ix86_avx256_split_vector_move_misalign (op0, op1);
      else
	/* Always use 128-bit mov<mode>_internal pattern for AVX.  */
	emit_insn (gen_rtx_SET (op0, op1));
      return;
    }

  if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
      || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
    {
      emit_insn (gen_rtx_SET (op0, op1));
      return;
    }

  /* ??? If we have typed data, then it would appear that using
     movdqu is the only way to get unaligned data loaded with
     integer type.  */
  if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
    {
      emit_insn (gen_rtx_SET (op0, op1));
      return;
    }

  if (MEM_P (op1))
    {
      if (TARGET_SSE2 && mode == V2DFmode)
        {
          rtx zero;

	  /* When SSE registers are split into halves, we can avoid
	     writing to the top half twice.  */
	  if (TARGET_SSE_SPLIT_REGS)
	    {
	      emit_clobber (op0);
	      zero = op0;
	    }
	  else
	    {
	      /* ??? Not sure about the best option for the Intel chips.
		 The following would seem to satisfy; the register is
		 entirely cleared, breaking the dependency chain.  We
		 then store to the upper half, with a dependency depth
		 of one.  A rumor has it that Intel recommends two movsd
		 followed by an unpacklpd, but this is unconfirmed.  And
		 given that the dependency depth of the unpacklpd would
		 still be one, I'm not sure why this would be better.  */
	      zero = CONST0_RTX (V2DFmode);
	    }

	  m = adjust_address (op1, DFmode, 0);
	  emit_insn (gen_sse2_loadlpd (op0, zero, m));
	  m = adjust_address (op1, DFmode, 8);
	  emit_insn (gen_sse2_loadhpd (op0, op0, m));
	}
      else
        {
	  rtx t;

	  if (mode != V4SFmode)
	    t = gen_reg_rtx (V4SFmode);
	  else
	    t = op0;
	    
	  if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
	    emit_move_insn (t, CONST0_RTX (V4SFmode));
	  else
	    emit_clobber (t);

	  m = adjust_address (op1, V2SFmode, 0);
	  emit_insn (gen_sse_loadlps (t, t, m));
	  m = adjust_address (op1, V2SFmode, 8);
	  emit_insn (gen_sse_loadhps (t, t, m));
	  if (mode != V4SFmode)
	    emit_move_insn (op0, gen_lowpart (mode, t));
	}
    }
  else if (MEM_P (op0))
    {
      if (TARGET_SSE2 && mode == V2DFmode)
	{
	  m = adjust_address (op0, DFmode, 0);
	  emit_insn (gen_sse2_storelpd (m, op1));
	  m = adjust_address (op0, DFmode, 8);
	  emit_insn (gen_sse2_storehpd (m, op1));
	}
      else
	{
	  if (mode != V4SFmode)
	    op1 = gen_lowpart (V4SFmode, op1);

	  m = adjust_address (op0, V2SFmode, 0);
	  emit_insn (gen_sse_storelps (m, op1));
	  m = adjust_address (op0, V2SFmode, 8);
	  emit_insn (gen_sse_storehps (m, copy_rtx (op1)));
	}
    }
  else
    gcc_unreachable ();
}

/* Helper function of ix86_fixup_binary_operands to canonicalize
   operand order.  Returns true if the operands should be swapped.  */

static bool
ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
			     rtx operands[])
{
  rtx dst = operands[0];
  rtx src1 = operands[1];
  rtx src2 = operands[2];

  /* If the operation is not commutative, we can't do anything.  */
  if (GET_RTX_CLASS (code) != RTX_COMM_ARITH)
    return false;

  /* Highest priority is that src1 should match dst.  */
  if (rtx_equal_p (dst, src1))
    return false;
  if (rtx_equal_p (dst, src2))
    return true;

  /* Next highest priority is that immediate constants come second.  */
  if (immediate_operand (src2, mode))
    return false;
  if (immediate_operand (src1, mode))
    return true;

  /* Lowest priority is that memory references should come second.  */
  if (MEM_P (src2))
    return false;
  if (MEM_P (src1))
    return true;

  return false;
}


/* Fix up OPERANDS to satisfy ix86_binary_operator_ok.  Return the
   destination to use for the operation.  If different from the true
   destination in operands[0], a copy operation will be required.  */

rtx
ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
			    rtx operands[])
{
  rtx dst = operands[0];
  rtx src1 = operands[1];
  rtx src2 = operands[2];

  /* Canonicalize operand order.  */
  if (ix86_swap_binary_operands_p (code, mode, operands))
    {
      /* It is invalid to swap operands of different modes.  */
      gcc_assert (GET_MODE (src1) == GET_MODE (src2));

      std::swap (src1, src2);
    }

  /* Both source operands cannot be in memory.  */
  if (MEM_P (src1) && MEM_P (src2))
    {
      /* Optimization: Only read from memory once.  */
      if (rtx_equal_p (src1, src2))
	{
	  src2 = force_reg (mode, src2);
	  src1 = src2;
	}
      else if (rtx_equal_p (dst, src1))
	src2 = force_reg (mode, src2);
      else
	src1 = force_reg (mode, src1);
    }

  /* If the destination is memory, and we do not have matching source
     operands, do things in registers.  */
  if (MEM_P (dst) && !rtx_equal_p (dst, src1))
    dst = gen_reg_rtx (mode);

  /* Source 1 cannot be a constant.  */
  if (CONSTANT_P (src1))
    src1 = force_reg (mode, src1);

  /* Source 1 cannot be a non-matching memory.  */
  if (MEM_P (src1) && !rtx_equal_p (dst, src1))
    src1 = force_reg (mode, src1);

  /* Improve address combine.  */
  if (code == PLUS
      && GET_MODE_CLASS (mode) == MODE_INT
      && MEM_P (src2))
    src2 = force_reg (mode, src2);

  operands[1] = src1;
  operands[2] = src2;
  return dst;
}

/* Similarly, but assume that the destination has already been
   set up properly.  */

void
ix86_fixup_binary_operands_no_copy (enum rtx_code code,
				    machine_mode mode, rtx operands[])
{
  rtx dst = ix86_fixup_binary_operands (code, mode, operands);
  gcc_assert (dst == operands[0]);
}

/* Attempt to expand a binary operator.  Make the expansion closer to the
   actual machine, then just general_operand, which will allow 3 separate
   memory references (one output, two input) in a single insn.  */

void
ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
			     rtx operands[])
{
  rtx src1, src2, dst, op, clob;

  dst = ix86_fixup_binary_operands (code, mode, operands);
  src1 = operands[1];
  src2 = operands[2];

 /* Emit the instruction.  */

  op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));

  if (reload_completed
      && code == PLUS
      && !rtx_equal_p (dst, src1))
    {
      /* This is going to be an LEA; avoid splitting it later.  */
      emit_insn (op);
    }
  else
    {
      clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
      emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
    }

  /* Fix up the destination if needed.  */
  if (dst != operands[0])
    emit_move_insn (operands[0], dst);
}

/* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
   the given OPERANDS.  */

void
ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
				     rtx operands[])
{
  rtx op1 = NULL_RTX, op2 = NULL_RTX;
  if (SUBREG_P (operands[1]))
    {
      op1 = operands[1];
      op2 = operands[2];
    }
  else if (SUBREG_P (operands[2]))
    {
      op1 = operands[2];
      op2 = operands[1];
    }
  /* Optimize (__m128i) d | (__m128i) e and similar code
     when d and e are float vectors into float vector logical
     insn.  In C/C++ without using intrinsics there is no other way
     to express vector logical operation on float vectors than
     to cast them temporarily to integer vectors.  */
  if (op1
      && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
      && (SUBREG_P (op2) || GET_CODE (op2) == CONST_VECTOR)
      && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
      && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
      && SUBREG_BYTE (op1) == 0
      && (GET_CODE (op2) == CONST_VECTOR
	  || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
	      && SUBREG_BYTE (op2) == 0))
      && can_create_pseudo_p ())
    {
      rtx dst;
      switch (GET_MODE (SUBREG_REG (op1)))
	{
	case V4SFmode:
	case V8SFmode:
	case V16SFmode:
	case V2DFmode:
	case V4DFmode:
	case V8DFmode:
	  dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
	  if (GET_CODE (op2) == CONST_VECTOR)
	    {
	      op2 = gen_lowpart (GET_MODE (dst), op2);
	      op2 = force_reg (GET_MODE (dst), op2);
	    }
	  else
	    {
	      op1 = operands[1];
	      op2 = SUBREG_REG (operands[2]);
	      if (!vector_operand (op2, GET_MODE (dst)))
		op2 = force_reg (GET_MODE (dst), op2);
	    }
	  op1 = SUBREG_REG (op1);
	  if (!vector_operand (op1, GET_MODE (dst)))
	    op1 = force_reg (GET_MODE (dst), op1);
	  emit_insn (gen_rtx_SET (dst,
				  gen_rtx_fmt_ee (code, GET_MODE (dst),
						  op1, op2)));
	  emit_move_insn (operands[0], gen_lowpart (mode, dst));
	  return;
	default:
	  break;
	}
    }
  if (!vector_operand (operands[1], mode))
    operands[1] = force_reg (mode, operands[1]);
  if (!vector_operand (operands[2], mode))
    operands[2] = force_reg (mode, operands[2]);
  ix86_fixup_binary_operands_no_copy (code, mode, operands);
  emit_insn (gen_rtx_SET (operands[0],
			  gen_rtx_fmt_ee (code, mode, operands[1],
					  operands[2])));
}

/* Return TRUE or FALSE depending on whether the binary operator meets the
   appropriate constraints.  */

bool
ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
			 rtx operands[3])
{
  rtx dst = operands[0];
  rtx src1 = operands[1];
  rtx src2 = operands[2];

  /* Both source operands cannot be in memory.  */
  if (MEM_P (src1) && MEM_P (src2))
    return false;

  /* Canonicalize operand order for commutative operators.  */
  if (ix86_swap_binary_operands_p (code, mode, operands))
    std::swap (src1, src2);

  /* If the destination is memory, we must have a matching source operand.  */
  if (MEM_P (dst) && !rtx_equal_p (dst, src1))
      return false;

  /* Source 1 cannot be a constant.  */
  if (CONSTANT_P (src1))
    return false;

  /* Source 1 cannot be a non-matching memory.  */
  if (MEM_P (src1) && !rtx_equal_p (dst, src1))
    /* Support "andhi/andsi/anddi" as a zero-extending move.  */
    return (code == AND
	    && (mode == HImode
		|| mode == SImode
		|| (TARGET_64BIT && mode == DImode))
	    && satisfies_constraint_L (src2));

  return true;
}

/* Attempt to expand a unary operator.  Make the expansion closer to the
   actual machine, then just general_operand, which will allow 2 separate
   memory references (one output, one input) in a single insn.  */

void
ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
			    rtx operands[])
{
  bool matching_memory = false;
  rtx src, dst, op, clob;

  dst = operands[0];
  src = operands[1];

  /* If the destination is memory, and we do not have matching source
     operands, do things in registers.  */
  if (MEM_P (dst))
    {
      if (rtx_equal_p (dst, src))
	matching_memory = true;
      else
	dst = gen_reg_rtx (mode);
    }

  /* When source operand is memory, destination must match.  */
  if (MEM_P (src) && !matching_memory)
    src = force_reg (mode, src);

  /* Emit the instruction.  */

  op = gen_rtx_SET (dst, gen_rtx_fmt_e (code, mode, src));

  if (code == NOT)
    emit_insn (op);
  else
    {
      clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
      emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
    }

  /* Fix up the destination if needed.  */
  if (dst != operands[0])
    emit_move_insn (operands[0], dst);
}

/* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
   divisor are within the range [0-255].  */

void
ix86_split_idivmod (machine_mode mode, rtx operands[],
		    bool signed_p)
{
  rtx_code_label *end_label, *qimode_label;
  rtx div, mod;
  rtx_insn *insn;
  rtx scratch, tmp0, tmp1, tmp2;
  rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
  rtx (*gen_zero_extend) (rtx, rtx);
  rtx (*gen_test_ccno_1) (rtx, rtx);

  switch (mode)
    {
    case SImode:
      gen_divmod4_1 = signed_p ? gen_divmodsi4_1 : gen_udivmodsi4_1;
      gen_test_ccno_1 = gen_testsi_ccno_1;
      gen_zero_extend = gen_zero_extendqisi2;
      break;
    case DImode:
      gen_divmod4_1 = signed_p ? gen_divmoddi4_1 : gen_udivmoddi4_1;
      gen_test_ccno_1 = gen_testdi_ccno_1;
      gen_zero_extend = gen_zero_extendqidi2;
      break;
    default:
      gcc_unreachable ();
    }

  end_label = gen_label_rtx ();
  qimode_label = gen_label_rtx ();

  scratch = gen_reg_rtx (mode);

  /* Use 8bit unsigned divimod if dividend and divisor are within
     the range [0-255].  */
  emit_move_insn (scratch, operands[2]);
  scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
				 scratch, 1, OPTAB_DIRECT);
  emit_insn (gen_test_ccno_1 (scratch, GEN_INT (-0x100)));
  tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
  tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
  tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
			       gen_rtx_LABEL_REF (VOIDmode, qimode_label),
			       pc_rtx);
  insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp0));
  predict_jump (REG_BR_PROB_BASE * 50 / 100);
  JUMP_LABEL (insn) = qimode_label;

  /* Generate original signed/unsigned divimod.  */
  div = gen_divmod4_1 (operands[0], operands[1],
		       operands[2], operands[3]);
  emit_insn (div);

  /* Branch to the end.  */
  emit_jump_insn (gen_jump (end_label));
  emit_barrier ();

  /* Generate 8bit unsigned divide.  */
  emit_label (qimode_label);
  /* Don't use operands[0] for result of 8bit divide since not all
     registers support QImode ZERO_EXTRACT.  */
  tmp0 = lowpart_subreg (HImode, scratch, mode);
  tmp1 = lowpart_subreg (HImode, operands[2], mode);
  tmp2 = lowpart_subreg (QImode, operands[3], mode);
  emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));

  if (signed_p)
    {
      div = gen_rtx_DIV (SImode, operands[2], operands[3]);
      mod = gen_rtx_MOD (SImode, operands[2], operands[3]);
    }
  else
    {
      div = gen_rtx_UDIV (SImode, operands[2], operands[3]);
      mod = gen_rtx_UMOD (SImode, operands[2], operands[3]);
    }

  /* Extract remainder from AH.  */
  tmp1 = gen_rtx_ZERO_EXTRACT (mode, tmp0, GEN_INT (8), GEN_INT (8));
  if (REG_P (operands[1]))
    insn = emit_move_insn (operands[1], tmp1);
  else
    {
      /* Need a new scratch register since the old one has result
	 of 8bit divide.  */
      scratch = gen_reg_rtx (mode);
      emit_move_insn (scratch, tmp1);
      insn = emit_move_insn (operands[1], scratch);
    }
  set_unique_reg_note (insn, REG_EQUAL, mod);

  /* Zero extend quotient from AL.  */
  tmp1 = gen_lowpart (QImode, tmp0);
  insn = emit_insn (gen_zero_extend (operands[0], tmp1));
  set_unique_reg_note (insn, REG_EQUAL, div);

  emit_label (end_label);
}

#define LEA_MAX_STALL (3)
#define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)

/* Increase given DISTANCE in half-cycles according to
   dependencies between PREV and NEXT instructions.
   Add 1 half-cycle if there is no dependency and
   go to next cycle if there is some dependecy.  */

static unsigned int
increase_distance (rtx_insn *prev, rtx_insn *next, unsigned int distance)
{
  df_ref def, use;

  if (!prev || !next)
    return distance + (distance & 1) + 2;

  if (!DF_INSN_USES (next) || !DF_INSN_DEFS (prev))
    return distance + 1;

  FOR_EACH_INSN_USE (use, next)
    FOR_EACH_INSN_DEF (def, prev)
      if (!DF_REF_IS_ARTIFICIAL (def)
	  && DF_REF_REGNO (use) == DF_REF_REGNO (def))
	return distance + (distance & 1) + 2;

  return distance + 1;
}

/* Function checks if instruction INSN defines register number
   REGNO1 or REGNO2.  */

static bool
insn_defines_reg (unsigned int regno1, unsigned int regno2,
		  rtx_insn *insn)
{
  df_ref def;

  FOR_EACH_INSN_DEF (def, insn)
    if (DF_REF_REG_DEF_P (def)
	&& !DF_REF_IS_ARTIFICIAL (def)
	&& (regno1 == DF_REF_REGNO (def)
	    || regno2 == DF_REF_REGNO (def)))
      return true;

  return false;
}

/* Function checks if instruction INSN uses register number
   REGNO as a part of address expression.  */

static bool
insn_uses_reg_mem (unsigned int regno, rtx insn)
{
  df_ref use;

  FOR_EACH_INSN_USE (use, insn)
    if (DF_REF_REG_MEM_P (use) && regno == DF_REF_REGNO (use))
      return true;

  return false;
}

/* Search backward for non-agu definition of register number REGNO1
   or register number REGNO2 in basic block starting from instruction
   START up to head of basic block or instruction INSN.

   Function puts true value into *FOUND var if definition was found
   and false otherwise.

   Distance in half-cycles between START and found instruction or head
   of BB is added to DISTANCE and returned.  */

static int
distance_non_agu_define_in_bb (unsigned int regno1, unsigned int regno2,
			       rtx_insn *insn, int distance,
			       rtx_insn *start, bool *found)
{
  basic_block bb = start ? BLOCK_FOR_INSN (start) : NULL;
  rtx_insn *prev = start;
  rtx_insn *next = NULL;

  *found = false;

  while (prev
	 && prev != insn
	 && distance < LEA_SEARCH_THRESHOLD)
    {
      if (NONDEBUG_INSN_P (prev) && NONJUMP_INSN_P (prev))
	{
	  distance = increase_distance (prev, next, distance);
	  if (insn_defines_reg (regno1, regno2, prev))
	    {
	      if (recog_memoized (prev) < 0
		  || get_attr_type (prev) != TYPE_LEA)
		{
		  *found = true;
		  return distance;
		}
	    }

	  next = prev;
	}
      if (prev == BB_HEAD (bb))
	break;

      prev = PREV_INSN (prev);
    }

  return distance;
}

/* Search backward for non-agu definition of register number REGNO1
   or register number REGNO2 in INSN's basic block until
   1. Pass LEA_SEARCH_THRESHOLD instructions, or
   2. Reach neighbor BBs boundary, or
   3. Reach agu definition.
   Returns the distance between the non-agu definition point and INSN.
   If no definition point, returns -1.  */

static int
distance_non_agu_define (unsigned int regno1, unsigned int regno2,
			 rtx_insn *insn)
{
  basic_block bb = BLOCK_FOR_INSN (insn);
  int distance = 0;
  bool found = false;

  if (insn != BB_HEAD (bb))
    distance = distance_non_agu_define_in_bb (regno1, regno2, insn,
					      distance, PREV_INSN (insn),
					      &found);

  if (!found && distance < LEA_SEARCH_THRESHOLD)
    {
      edge e;
      edge_iterator ei;
      bool simple_loop = false;

      FOR_EACH_EDGE (e, ei, bb->preds)
	if (e->src == bb)
	  {
	    simple_loop = true;
	    break;
	  }

      if (simple_loop)
	distance = distance_non_agu_define_in_bb (regno1, regno2,
						  insn, distance,
						  BB_END (bb), &found);
      else
	{
	  int shortest_dist = -1;
	  bool found_in_bb = false;

	  FOR_EACH_EDGE (e, ei, bb->preds)
	    {
	      int bb_dist
		= distance_non_agu_define_in_bb (regno1, regno2,
						 insn, distance,
						 BB_END (e->src),
						 &found_in_bb);
	      if (found_in_bb)
		{
		  if (shortest_dist < 0)
		    shortest_dist = bb_dist;
		  else if (bb_dist > 0)
		    shortest_dist = MIN (bb_dist, shortest_dist);

		  found = true;
		}
	    }

	  distance = shortest_dist;
	}
    }

  /* get_attr_type may modify recog data.  We want to make sure
     that recog data is valid for instruction INSN, on which
     distance_non_agu_define is called.  INSN is unchanged here.  */
  extract_insn_cached (insn);

  if (!found)
    return -1;

  return distance >> 1;
}

/* Return the distance in half-cycles between INSN and the next
   insn that uses register number REGNO in memory address added
   to DISTANCE.  Return -1 if REGNO0 is set.

   Put true value into *FOUND if register usage was found and
   false otherwise.
   Put true value into *REDEFINED if register redefinition was
   found and false otherwise.  */

static int
distance_agu_use_in_bb (unsigned int regno,
			rtx_insn *insn, int distance, rtx_insn *start,
			bool *found, bool *redefined)
{
  basic_block bb = NULL;
  rtx_insn *next = start;
  rtx_insn *prev = NULL;

  *found = false;
  *redefined = false;

  if (start != NULL_RTX)
    {
      bb = BLOCK_FOR_INSN (start);
      if (start != BB_HEAD (bb))
	/* If insn and start belong to the same bb, set prev to insn,
	   so the call to increase_distance will increase the distance
	   between insns by 1.  */
	prev = insn;
    }

  while (next
	 && next != insn
	 && distance < LEA_SEARCH_THRESHOLD)
    {
      if (NONDEBUG_INSN_P (next) && NONJUMP_INSN_P (next))
	{
	  distance = increase_distance(prev, next, distance);
	  if (insn_uses_reg_mem (regno, next))
	    {
	      /* Return DISTANCE if OP0 is used in memory
		 address in NEXT.  */
	      *found = true;
	      return distance;
	    }

	  if (insn_defines_reg (regno, INVALID_REGNUM, next))
	    {
	      /* Return -1 if OP0 is set in NEXT.  */
	      *redefined = true;
	      return -1;
	    }

	  prev = next;
	}

      if (next == BB_END (bb))
	break;

      next = NEXT_INSN (next);
    }

  return distance;
}

/* Return the distance between INSN and the next insn that uses
   register number REGNO0 in memory address.  Return -1 if no such
   a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set.  */

static int
distance_agu_use (unsigned int regno0, rtx_insn *insn)
{
  basic_block bb = BLOCK_FOR_INSN (insn);
  int distance = 0;
  bool found = false;
  bool redefined = false;

  if (insn != BB_END (bb))
    distance = distance_agu_use_in_bb (regno0, insn, distance,
				       NEXT_INSN (insn),
				       &found, &redefined);

  if (!found && !redefined && distance < LEA_SEARCH_THRESHOLD)
    {
      edge e;
      edge_iterator ei;
      bool simple_loop = false;

      FOR_EACH_EDGE (e, ei, bb->succs)
        if (e->dest == bb)
	  {
	    simple_loop = true;
	    break;
	  }

      if (simple_loop)
	distance = distance_agu_use_in_bb (regno0, insn,
					   distance, BB_HEAD (bb),
					   &found, &redefined);
      else
	{
	  int shortest_dist = -1;
	  bool found_in_bb = false;
	  bool redefined_in_bb = false;

	  FOR_EACH_EDGE (e, ei, bb->succs)
	    {
	      int bb_dist
		= distance_agu_use_in_bb (regno0, insn,
					  distance, BB_HEAD (e->dest),
					  &found_in_bb, &redefined_in_bb);
	      if (found_in_bb)
		{
		  if (shortest_dist < 0)
		    shortest_dist = bb_dist;
		  else if (bb_dist > 0)
		    shortest_dist = MIN (bb_dist, shortest_dist);

		  found = true;
		}
	    }

	  distance = shortest_dist;
	}
    }

  if (!found || redefined)
    return -1;

  return distance >> 1;
}

/* Define this macro to tune LEA priority vs ADD, it take effect when
   there is a dilemma of choicing LEA or ADD
   Negative value: ADD is more preferred than LEA
   Zero: Netrual
   Positive value: LEA is more preferred than ADD*/
#define IX86_LEA_PRIORITY 0

/* Return true if usage of lea INSN has performance advantage
   over a sequence of instructions.  Instructions sequence has
   SPLIT_COST cycles higher latency than lea latency.  */

static bool
ix86_lea_outperforms (rtx_insn *insn, unsigned int regno0, unsigned int regno1,
		      unsigned int regno2, int split_cost, bool has_scale)
{
  int dist_define, dist_use;

  /* For Silvermont if using a 2-source or 3-source LEA for
     non-destructive destination purposes, or due to wanting
     ability to use SCALE, the use of LEA is justified.  */
  if (TARGET_SILVERMONT || TARGET_INTEL)
    {
      if (has_scale)
	return true;
      if (split_cost < 1)
	return false;
      if (regno0 == regno1 || regno0 == regno2)
	return false;
      return true;
    }

  dist_define = distance_non_agu_define (regno1, regno2, insn);
  dist_use = distance_agu_use (regno0, insn);

  if (dist_define < 0 || dist_define >= LEA_MAX_STALL)
    {
      /* If there is no non AGU operand definition, no AGU
	 operand usage and split cost is 0 then both lea
	 and non lea variants have same priority.  Currently
	 we prefer lea for 64 bit code and non lea on 32 bit
	 code.  */
      if (dist_use < 0 && split_cost == 0)
	return TARGET_64BIT || IX86_LEA_PRIORITY;
      else
	return true;
    }

  /* With longer definitions distance lea is more preferable.
     Here we change it to take into account splitting cost and
     lea priority.  */
  dist_define += split_cost + IX86_LEA_PRIORITY;

  /* If there is no use in memory addess then we just check
     that split cost exceeds AGU stall.  */
  if (dist_use < 0)
    return dist_define > LEA_MAX_STALL;

  /* If this insn has both backward non-agu dependence and forward
     agu dependence, the one with short distance takes effect.  */
  return dist_define >= dist_use;
}

/* Return true if it is legal to clobber flags by INSN and
   false otherwise.  */

static bool
ix86_ok_to_clobber_flags (rtx_insn *insn)
{
  basic_block bb = BLOCK_FOR_INSN (insn);
  df_ref use;
  bitmap live;

  while (insn)
    {
      if (NONDEBUG_INSN_P (insn))
	{
	  FOR_EACH_INSN_USE (use, insn)
	    if (DF_REF_REG_USE_P (use) && DF_REF_REGNO (use) == FLAGS_REG)
	      return false;

	  if (insn_defines_reg (FLAGS_REG, INVALID_REGNUM, insn))
	    return true;
	}

      if (insn == BB_END (bb))
	break;

      insn = NEXT_INSN (insn);
    }

  live = df_get_live_out(bb);
  return !REGNO_REG_SET_P (live, FLAGS_REG);
}

/* Return true if we need to split op0 = op1 + op2 into a sequence of
   move and add to avoid AGU stalls.  */

bool
ix86_avoid_lea_for_add (rtx_insn *insn, rtx operands[])
{
  unsigned int regno0, regno1, regno2;

  /* Check if we need to optimize.  */
  if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
    return false;

  /* Check it is correct to split here.  */
  if (!ix86_ok_to_clobber_flags(insn))
    return false;

  regno0 = true_regnum (operands[0]);
  regno1 = true_regnum (operands[1]);
  regno2 = true_regnum (operands[2]);

  /* We need to split only adds with non destructive
     destination operand.  */
  if (regno0 == regno1 || regno0 == regno2)
    return false;
  else
    return !ix86_lea_outperforms (insn, regno0, regno1, regno2, 1, false);
}

/* Return true if we should emit lea instruction instead of mov
   instruction.  */

bool
ix86_use_lea_for_mov (rtx_insn *insn, rtx operands[])
{
  unsigned int regno0, regno1;

  /* Check if we need to optimize.  */
  if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
    return false;

  /* Use lea for reg to reg moves only.  */
  if (!REG_P (operands[0]) || !REG_P (operands[1]))
    return false;

  regno0 = true_regnum (operands[0]);
  regno1 = true_regnum (operands[1]);

  return ix86_lea_outperforms (insn, regno0, regno1, INVALID_REGNUM, 0, false);
}

/* Return true if we need to split lea into a sequence of
   instructions to avoid AGU stalls. */

bool
ix86_avoid_lea_for_addr (rtx_insn *insn, rtx operands[])
{
  unsigned int regno0, regno1, regno2;
  int split_cost;
  struct ix86_address parts;
  int ok;

  /* Check we need to optimize.  */
  if (!TARGET_AVOID_LEA_FOR_ADDR || optimize_function_for_size_p (cfun))
    return false;

  /* The "at least two components" test below might not catch simple
     move or zero extension insns if parts.base is non-NULL and parts.disp
     is const0_rtx as the only components in the address, e.g. if the
     register is %rbp or %r13.  As this test is much cheaper and moves or
     zero extensions are the common case, do this check first.  */
  if (REG_P (operands[1])
      || (SImode_address_operand (operands[1], VOIDmode)
	  && REG_P (XEXP (operands[1], 0))))
    return false;

  /* Check if it is OK to split here.  */
  if (!ix86_ok_to_clobber_flags (insn))
    return false;

  ok = ix86_decompose_address (operands[1], &parts);
  gcc_assert (ok);

  /* There should be at least two components in the address.  */
  if ((parts.base != NULL_RTX) + (parts.index != NULL_RTX)
      + (parts.disp != NULL_RTX) + (parts.scale > 1) < 2)
    return false;

  /* We should not split into add if non legitimate pic
     operand is used as displacement. */
  if (parts.disp && flag_pic && !LEGITIMATE_PIC_OPERAND_P (parts.disp))
    return false;

  regno0 = true_regnum (operands[0]) ;
  regno1 = INVALID_REGNUM;
  regno2 = INVALID_REGNUM;

  if (parts.base)
    regno1 = true_regnum (parts.base);
  if (parts.index)
    regno2 = true_regnum (parts.index);

  split_cost = 0;

  /* Compute how many cycles we will add to execution time
     if split lea into a sequence of instructions.  */
  if (parts.base || parts.index)
    {
      /* Have to use mov instruction if non desctructive
	 destination form is used.  */
      if (regno1 != regno0 && regno2 != regno0)
	split_cost += 1;

      /* Have to add index to base if both exist.  */
      if (parts.base && parts.index)
	split_cost += 1;

      /* Have to use shift and adds if scale is 2 or greater.  */
      if (parts.scale > 1)
	{
	  if (regno0 != regno1)
	    split_cost += 1;
	  else if (regno2 == regno0)
	    split_cost += 4;
	  else
	    split_cost += parts.scale;
	}

      /* Have to use add instruction with immediate if
	 disp is non zero.  */
      if (parts.disp && parts.disp != const0_rtx)
	split_cost += 1;

      /* Subtract the price of lea.  */
      split_cost -= 1;
    }

  return !ix86_lea_outperforms (insn, regno0, regno1, regno2, split_cost,
				parts.scale > 1);
}

/* Emit x86 binary operand CODE in mode MODE, where the first operand
   matches destination.  RTX includes clobber of FLAGS_REG.  */

static void
ix86_emit_binop (enum rtx_code code, machine_mode mode,
		 rtx dst, rtx src)
{
  rtx op, clob;

  op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, dst, src));
  clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
  
  emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
}

/* Return true if regno1 def is nearest to the insn.  */

static bool
find_nearest_reg_def (rtx_insn *insn, int regno1, int regno2)
{
  rtx_insn *prev = insn;
  rtx_insn *start = BB_HEAD (BLOCK_FOR_INSN (insn));

  if (insn == start)
    return false;
  while (prev && prev != start)
    {
      if (!INSN_P (prev) || !NONDEBUG_INSN_P (prev))
	{
	  prev = PREV_INSN (prev);
	  continue;
	}
      if (insn_defines_reg (regno1, INVALID_REGNUM, prev))
	return true;
      else if (insn_defines_reg (regno2, INVALID_REGNUM, prev))
	return false;
      prev = PREV_INSN (prev);
    }

  /* None of the regs is defined in the bb.  */
  return false;
}

/* Split lea instructions into a sequence of instructions
   which are executed on ALU to avoid AGU stalls.
   It is assumed that it is allowed to clobber flags register
   at lea position.  */

void
ix86_split_lea_for_addr (rtx_insn *insn, rtx operands[], machine_mode mode)
{
  unsigned int regno0, regno1, regno2;
  struct ix86_address parts;
  rtx target, tmp;
  int ok, adds;

  ok = ix86_decompose_address (operands[1], &parts);
  gcc_assert (ok);

  target = gen_lowpart (mode, operands[0]);

  regno0 = true_regnum (target);
  regno1 = INVALID_REGNUM;
  regno2 = INVALID_REGNUM;

  if (parts.base)
    {
      parts.base = gen_lowpart (mode, parts.base);
      regno1 = true_regnum (parts.base);
    }

  if (parts.index)
    {
      parts.index = gen_lowpart (mode, parts.index);
      regno2 = true_regnum (parts.index);
    }

  if (parts.disp)
    parts.disp = gen_lowpart (mode, parts.disp);

  if (parts.scale > 1)
    {
      /* Case r1 = r1 + ...  */
      if (regno1 == regno0)
	{
	  /* If we have a case r1 = r1 + C * r2 then we
	     should use multiplication which is very
	     expensive.  Assume cost model is wrong if we
	     have such case here.  */
	  gcc_assert (regno2 != regno0);

	  for (adds = parts.scale; adds > 0; adds--)
	    ix86_emit_binop (PLUS, mode, target, parts.index);
	}
      else
	{
	  /* r1 = r2 + r3 * C case.  Need to move r3 into r1.  */
	  if (regno0 != regno2)
	    emit_insn (gen_rtx_SET (target, parts.index));

	  /* Use shift for scaling.  */
	  ix86_emit_binop (ASHIFT, mode, target,
			   GEN_INT (exact_log2 (parts.scale)));

	  if (parts.base)
	    ix86_emit_binop (PLUS, mode, target, parts.base);

	  if (parts.disp && parts.disp != const0_rtx)
	    ix86_emit_binop (PLUS, mode, target, parts.disp);
	}
    }
  else if (!parts.base && !parts.index)
    {
      gcc_assert(parts.disp);
      emit_insn (gen_rtx_SET (target, parts.disp));
    }
  else
    {
      if (!parts.base)
	{
	  if (regno0 != regno2)
	    emit_insn (gen_rtx_SET (target, parts.index));
	}
      else if (!parts.index)
	{
	  if (regno0 != regno1)
	    emit_insn (gen_rtx_SET (target, parts.base));
	}
      else
	{
	  if (regno0 == regno1)
	    tmp = parts.index;
	  else if (regno0 == regno2)
	    tmp = parts.base;
	  else
	    {
	      rtx tmp1;

	      /* Find better operand for SET instruction, depending
		 on which definition is farther from the insn.  */
	      if (find_nearest_reg_def (insn, regno1, regno2))
		tmp = parts.index, tmp1 = parts.base;
	      else
		tmp = parts.base, tmp1 = parts.index;

	      emit_insn (gen_rtx_SET (target, tmp));

	      if (parts.disp && parts.disp != const0_rtx)
		ix86_emit_binop (PLUS, mode, target, parts.disp);

	      ix86_emit_binop (PLUS, mode, target, tmp1);
	      return;
	    }

	  ix86_emit_binop (PLUS, mode, target, tmp);
	}

      if (parts.disp && parts.disp != const0_rtx)
	ix86_emit_binop (PLUS, mode, target, parts.disp);
    }
}

/* Return true if it is ok to optimize an ADD operation to LEA
   operation to avoid flag register consumation.  For most processors,
   ADD is faster than LEA.  For the processors like BONNELL, if the
   destination register of LEA holds an actual address which will be
   used soon, LEA is better and otherwise ADD is better.  */

bool
ix86_lea_for_add_ok (rtx_insn *insn, rtx operands[])
{
  unsigned int regno0 = true_regnum (operands[0]);
  unsigned int regno1 = true_regnum (operands[1]);
  unsigned int regno2 = true_regnum (operands[2]);

  /* If a = b + c, (a!=b && a!=c), must use lea form. */
  if (regno0 != regno1 && regno0 != regno2)
    return true;

  if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
    return false;

  return ix86_lea_outperforms (insn, regno0, regno1, regno2, 0, false);
}

/* Return true if destination reg of SET_BODY is shift count of
   USE_BODY.  */

static bool
ix86_dep_by_shift_count_body (const_rtx set_body, const_rtx use_body)
{
  rtx set_dest;
  rtx shift_rtx;
  int i;

  /* Retrieve destination of SET_BODY.  */
  switch (GET_CODE (set_body))
    {
    case SET:
      set_dest = SET_DEST (set_body);
      if (!set_dest || !REG_P (set_dest))
	return false;
      break;
    case PARALLEL:
      for (i = XVECLEN (set_body, 0) - 1; i >= 0; i--)
	if (ix86_dep_by_shift_count_body (XVECEXP (set_body, 0, i),
					  use_body))
	  return true;
      /* FALLTHROUGH */
    default:
      return false;
    }

  /* Retrieve shift count of USE_BODY.  */
  switch (GET_CODE (use_body))
    {
    case SET:
      shift_rtx = XEXP (use_body, 1);
      break;
    case PARALLEL:
      for (i = XVECLEN (use_body, 0) - 1; i >= 0; i--)
	if (ix86_dep_by_shift_count_body (set_body,
					  XVECEXP (use_body, 0, i)))
	  return true;
      /* FALLTHROUGH */
    default:
      return false;
    }

  if (shift_rtx
      && (GET_CODE (shift_rtx) == ASHIFT
	  || GET_CODE (shift_rtx) == LSHIFTRT
	  || GET_CODE (shift_rtx) == ASHIFTRT
	  || GET_CODE (shift_rtx) == ROTATE
	  || GET_CODE (shift_rtx) == ROTATERT))
    {
      rtx shift_count = XEXP (shift_rtx, 1);

      /* Return true if shift count is dest of SET_BODY.  */
      if (REG_P (shift_count))
	{
	  /* Add check since it can be invoked before register
	     allocation in pre-reload schedule.  */
	  if (reload_completed
	      && true_regnum (set_dest) == true_regnum (shift_count))
	    return true;
	  else if (REGNO(set_dest) == REGNO(shift_count))
	    return true;
	}
    }

  return false;
}

/* Return true if destination reg of SET_INSN is shift count of
   USE_INSN.  */

bool
ix86_dep_by_shift_count (const_rtx set_insn, const_rtx use_insn)
{
  return ix86_dep_by_shift_count_body (PATTERN (set_insn),
				       PATTERN (use_insn));
}

/* Return TRUE or FALSE depending on whether the unary operator meets the
   appropriate constraints.  */

bool
ix86_unary_operator_ok (enum rtx_code,
			machine_mode,
			rtx operands[2])
{
  /* If one of operands is memory, source and destination must match.  */
  if ((MEM_P (operands[0])
       || MEM_P (operands[1]))
      && ! rtx_equal_p (operands[0], operands[1]))
    return false;
  return true;
}

/* Return TRUE if the operands to a vec_interleave_{high,low}v2df
   are ok, keeping in mind the possible movddup alternative.  */

bool
ix86_vec_interleave_v2df_operator_ok (rtx operands[3], bool high)
{
  if (MEM_P (operands[0]))
    return rtx_equal_p (operands[0], operands[1 + high]);
  if (MEM_P (operands[1]) && MEM_P (operands[2]))
    return TARGET_SSE3 && rtx_equal_p (operands[1], operands[2]);
  return true;
}

/* Post-reload splitter for converting an SF or DFmode value in an
   SSE register into an unsigned SImode.  */

void
ix86_split_convert_uns_si_sse (rtx operands[])
{
  machine_mode vecmode;
  rtx value, large, zero_or_two31, input, two31, x;

  large = operands[1];
  zero_or_two31 = operands[2];
  input = operands[3];
  two31 = operands[4];
  vecmode = GET_MODE (large);
  value = gen_rtx_REG (vecmode, REGNO (operands[0]));

  /* Load up the value into the low element.  We must ensure that the other
     elements are valid floats -- zero is the easiest such value.  */
  if (MEM_P (input))
    {
      if (vecmode == V4SFmode)
	emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
      else
	emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
    }
  else
    {
      input = gen_rtx_REG (vecmode, REGNO (input));
      emit_move_insn (value, CONST0_RTX (vecmode));
      if (vecmode == V4SFmode)
	emit_insn (gen_sse_movss (value, value, input));
      else
	emit_insn (gen_sse2_movsd (value, value, input));
    }

  emit_move_insn (large, two31);
  emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);

  x = gen_rtx_fmt_ee (LE, vecmode, large, value);
  emit_insn (gen_rtx_SET (large, x));

  x = gen_rtx_AND (vecmode, zero_or_two31, large);
  emit_insn (gen_rtx_SET (zero_or_two31, x));

  x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
  emit_insn (gen_rtx_SET (value, x));

  large = gen_rtx_REG (V4SImode, REGNO (large));
  emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));

  x = gen_rtx_REG (V4SImode, REGNO (value));
  if (vecmode == V4SFmode)
    emit_insn (gen_fix_truncv4sfv4si2 (x, value));
  else
    emit_insn (gen_sse2_cvttpd2dq (x, value));
  value = x;

  emit_insn (gen_xorv4si3 (value, value, large));
}

/* Convert an unsigned DImode value into a DFmode, using only SSE.
   Expects the 64-bit DImode to be supplied in a pair of integral
   registers.  Requires SSE2; will use SSE3 if available.  For x86_32,
   -mfpmath=sse, !optimize_size only.  */

void
ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
{
  REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
  rtx int_xmm, fp_xmm;
  rtx biases, exponents;
  rtx x;

  int_xmm = gen_reg_rtx (V4SImode);
  if (TARGET_INTER_UNIT_MOVES_TO_VEC)
    emit_insn (gen_movdi_to_sse (int_xmm, input));
  else if (TARGET_SSE_SPLIT_REGS)
    {
      emit_clobber (int_xmm);
      emit_move_insn (gen_lowpart (DImode, int_xmm), input);
    }
  else
    {
      x = gen_reg_rtx (V2DImode);
      ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
      emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
    }

  x = gen_rtx_CONST_VECTOR (V4SImode,
			    gen_rtvec (4, GEN_INT (0x43300000UL),
				       GEN_INT (0x45300000UL),
				       const0_rtx, const0_rtx));
  exponents = validize_mem (force_const_mem (V4SImode, x));

  /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
  emit_insn (gen_vec_interleave_lowv4si (int_xmm, int_xmm, exponents));

  /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
     yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
     Similarly (0x45300000UL ## fp_value_hi_xmm) yields
     (0x1.0p84 + double(fp_value_hi_xmm)).
     Note these exponents differ by 32.  */

  fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));

  /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
     in [0,2**32-1] and [0]+[2**32,2**64-1] respectively.  */
  real_ldexp (&bias_lo_rvt, &dconst1, 52);
  real_ldexp (&bias_hi_rvt, &dconst1, 84);
  biases = const_double_from_real_value (bias_lo_rvt, DFmode);
  x = const_double_from_real_value (bias_hi_rvt, DFmode);
  biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
  biases = validize_mem (force_const_mem (V2DFmode, biases));
  emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));

  /* Add the upper and lower DFmode values together.  */
  if (TARGET_SSE3)
    emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
  else
    {
      x = copy_to_mode_reg (V2DFmode, fp_xmm);
      emit_insn (gen_vec_interleave_highv2df (fp_xmm, fp_xmm, fp_xmm));
      emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
    }

  ix86_expand_vector_extract (false, target, fp_xmm, 0);
}

/* Not used, but eases macroization of patterns.  */
void
ix86_expand_convert_uns_sixf_sse (rtx, rtx)
{
  gcc_unreachable ();
}

/* Convert an unsigned SImode value into a DFmode.  Only currently used
   for SSE, but applicable anywhere.  */

void
ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
{
  REAL_VALUE_TYPE TWO31r;
  rtx x, fp;

  x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
			   NULL, 1, OPTAB_DIRECT);

  fp = gen_reg_rtx (DFmode);
  emit_insn (gen_floatsidf2 (fp, x));

  real_ldexp (&TWO31r, &dconst1, 31);
  x = const_double_from_real_value (TWO31r, DFmode);

  x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
  if (x != target)
    emit_move_insn (target, x);
}

/* Convert a signed DImode value into a DFmode.  Only used for SSE in
   32-bit mode; otherwise we have a direct convert instruction.  */

void
ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
{
  REAL_VALUE_TYPE TWO32r;
  rtx fp_lo, fp_hi, x;

  fp_lo = gen_reg_rtx (DFmode);
  fp_hi = gen_reg_rtx (DFmode);

  emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));

  real_ldexp (&TWO32r, &dconst1, 32);
  x = const_double_from_real_value (TWO32r, DFmode);
  fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);

  ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));

  x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
			   0, OPTAB_DIRECT);
  if (x != target)
    emit_move_insn (target, x);
}

/* Convert an unsigned SImode value into a SFmode, using only SSE.
   For x86_32, -mfpmath=sse, !optimize_size only.  */
void
ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
{
  REAL_VALUE_TYPE ONE16r;
  rtx fp_hi, fp_lo, int_hi, int_lo, x;

  real_ldexp (&ONE16r, &dconst1, 16);
  x = const_double_from_real_value (ONE16r, SFmode);
  int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
				      NULL, 0, OPTAB_DIRECT);
  int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
				      NULL, 0, OPTAB_DIRECT);
  fp_hi = gen_reg_rtx (SFmode);
  fp_lo = gen_reg_rtx (SFmode);
  emit_insn (gen_floatsisf2 (fp_hi, int_hi));
  emit_insn (gen_floatsisf2 (fp_lo, int_lo));
  fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
			       0, OPTAB_DIRECT);
  fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
			       0, OPTAB_DIRECT);
  if (!rtx_equal_p (target, fp_hi))
    emit_move_insn (target, fp_hi);
}

/* floatunsv{4,8}siv{4,8}sf2 expander.  Expand code to convert
   a vector of unsigned ints VAL to vector of floats TARGET.  */

void
ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
{
  rtx tmp[8];
  REAL_VALUE_TYPE TWO16r;
  machine_mode intmode = GET_MODE (val);
  machine_mode fltmode = GET_MODE (target);
  rtx (*cvt) (rtx, rtx);

  if (intmode == V4SImode)
    cvt = gen_floatv4siv4sf2;
  else
    cvt = gen_floatv8siv8sf2;
  tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
  tmp[0] = force_reg (intmode, tmp[0]);
  tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
				OPTAB_DIRECT);
  tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
				NULL_RTX, 1, OPTAB_DIRECT);
  tmp[3] = gen_reg_rtx (fltmode);
  emit_insn (cvt (tmp[3], tmp[1]));
  tmp[4] = gen_reg_rtx (fltmode);
  emit_insn (cvt (tmp[4], tmp[2]));
  real_ldexp (&TWO16r, &dconst1, 16);
  tmp[5] = const_double_from_real_value (TWO16r, SFmode);
  tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
  tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5], NULL_RTX, 1,
				OPTAB_DIRECT);
  tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6], target, 1,
				OPTAB_DIRECT);
  if (tmp[7] != target)
    emit_move_insn (target, tmp[7]);
}

/* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
   pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
   This is done by doing just signed conversion if < 0x1p31, and otherwise by
   subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards.  */

rtx
ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
{
  REAL_VALUE_TYPE TWO31r;
  rtx two31r, tmp[4];
  machine_mode mode = GET_MODE (val);
  machine_mode scalarmode = GET_MODE_INNER (mode);
  machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
  rtx (*cmp) (rtx, rtx, rtx, rtx);
  int i;

  for (i = 0; i < 3; i++)
    tmp[i] = gen_reg_rtx (mode);
  real_ldexp (&TWO31r, &dconst1, 31);
  two31r = const_double_from_real_value (TWO31r, scalarmode);
  two31r = ix86_build_const_vector (mode, 1, two31r);
  two31r = force_reg (mode, two31r);
  switch (mode)
    {
    case V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
    case V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
    case V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
    case V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
    default: gcc_unreachable ();
    }
  tmp[3] = gen_rtx_LE (mode, two31r, val);
  emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
  tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
				0, OPTAB_DIRECT);
  if (intmode == V4SImode || TARGET_AVX2)
    *xorp = expand_simple_binop (intmode, ASHIFT,
				 gen_lowpart (intmode, tmp[0]),
				 GEN_INT (31), NULL_RTX, 0,
				 OPTAB_DIRECT);
  else
    {
      rtx two31 = GEN_INT (HOST_WIDE_INT_1U << 31);
      two31 = ix86_build_const_vector (intmode, 1, two31);
      *xorp = expand_simple_binop (intmode, AND,
				   gen_lowpart (intmode, tmp[0]),
				   two31, NULL_RTX, 0,
				   OPTAB_DIRECT);
    }
  return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
			      0, OPTAB_DIRECT);
}

/* A subroutine of ix86_build_signbit_mask.  If VECT is true,
   then replicate the value for all elements of the vector
   register.  */

rtx
ix86_build_const_vector (machine_mode mode, bool vect, rtx value)
{
  int i, n_elt;
  rtvec v;
  machine_mode scalar_mode;

  switch (mode)
    {
    case V64QImode:
    case V32QImode:
    case V16QImode:
    case V32HImode:
    case V16HImode:
    case V8HImode:
    case V16SImode:
    case V8SImode:
    case V4SImode:
    case V8DImode:
    case V4DImode:
    case V2DImode:
      gcc_assert (vect);
      /* FALLTHRU */
    case V16SFmode:
    case V8SFmode:
    case V4SFmode:
    case V8DFmode:
    case V4DFmode:
    case V2DFmode:
      n_elt = GET_MODE_NUNITS (mode);
      v = rtvec_alloc (n_elt);
      scalar_mode = GET_MODE_INNER (mode);

      RTVEC_ELT (v, 0) = value;

      for (i = 1; i < n_elt; ++i)
	RTVEC_ELT (v, i) = vect ? value : CONST0_RTX (scalar_mode);

      return gen_rtx_CONST_VECTOR (mode, v);

    default:
      gcc_unreachable ();
    }
}

/* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
   and ix86_expand_int_vcond.  Create a mask for the sign bit in MODE
   for an SSE register.  If VECT is true, then replicate the mask for
   all elements of the vector register.  If INVERT is true, then create
   a mask excluding the sign bit.  */

rtx
ix86_build_signbit_mask (machine_mode mode, bool vect, bool invert)
{
  machine_mode vec_mode, imode;
  wide_int w;
  rtx mask, v;

  switch (mode)
    {
    case V16SImode:
    case V16SFmode:
    case V8SImode:
    case V4SImode:
    case V8SFmode:
    case V4SFmode:
      vec_mode = mode;
      imode = SImode;
      break;

    case V8DImode:
    case V4DImode:
    case V2DImode:
    case V8DFmode:
    case V4DFmode:
    case V2DFmode:
      vec_mode = mode;
      imode = DImode;
      break;

    case TImode:
    case TFmode:
      vec_mode = VOIDmode;
      imode = TImode;
      break;

    default:
      gcc_unreachable ();
    }

  machine_mode inner_mode = GET_MODE_INNER (mode);
  w = wi::set_bit_in_zero (GET_MODE_BITSIZE (inner_mode) - 1,
			   GET_MODE_BITSIZE (inner_mode));
  if (invert)
    w = wi::bit_not (w);

  /* Force this value into the low part of a fp vector constant.  */
  mask = immed_wide_int_const (w, imode);
  mask = gen_lowpart (inner_mode, mask);

  if (vec_mode == VOIDmode)
    return force_reg (inner_mode, mask);

  v = ix86_build_const_vector (vec_mode, vect, mask);
  return force_reg (vec_mode, v);
}

/* Generate code for floating point ABS or NEG.  */

void
ix86_expand_fp_absneg_operator (enum rtx_code code, machine_mode mode,
				rtx operands[])
{
  rtx mask, set, dst, src;
  bool use_sse = false;
  bool vector_mode = VECTOR_MODE_P (mode);
  machine_mode vmode = mode;

  if (vector_mode)
    use_sse = true;
  else if (mode == TFmode)
    use_sse = true;
  else if (TARGET_SSE_MATH)
    {
      use_sse = SSE_FLOAT_MODE_P (mode);
      if (mode == SFmode)
	vmode = V4SFmode;
      else if (mode == DFmode)
	vmode = V2DFmode;
    }

  /* NEG and ABS performed with SSE use bitwise mask operations.
     Create the appropriate mask now.  */
  if (use_sse)
    mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
  else
    mask = NULL_RTX;

  dst = operands[0];
  src = operands[1];

  set = gen_rtx_fmt_e (code, mode, src);
  set = gen_rtx_SET (dst, set);

  if (mask)
    {
      rtx use, clob;
      rtvec par;

      use = gen_rtx_USE (VOIDmode, mask);
      if (vector_mode)
	par = gen_rtvec (2, set, use);
      else
	{
          clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
	  par = gen_rtvec (3, set, use, clob);
        }
      emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
    }
  else
    emit_insn (set);
}

/* Expand a copysign operation.  Special case operand 0 being a constant.  */

void
ix86_expand_copysign (rtx operands[])
{
  machine_mode mode, vmode;
  rtx dest, op0, op1, mask, nmask;

  dest = operands[0];
  op0 = operands[1];
  op1 = operands[2];

  mode = GET_MODE (dest);

  if (mode == SFmode)
    vmode = V4SFmode;
  else if (mode == DFmode)
    vmode = V2DFmode;
  else
    vmode = mode;

  if (CONST_DOUBLE_P (op0))
    {
      rtx (*copysign_insn)(rtx, rtx, rtx, rtx);

      if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
	op0 = simplify_unary_operation (ABS, mode, op0, mode);

      if (mode == SFmode || mode == DFmode)
	{
	  if (op0 == CONST0_RTX (mode))
	    op0 = CONST0_RTX (vmode);
	  else
	    {
	      rtx v = ix86_build_const_vector (vmode, false, op0);

	      op0 = force_reg (vmode, v);
	    }
	}
      else if (op0 != CONST0_RTX (mode))
	op0 = force_reg (mode, op0);

      mask = ix86_build_signbit_mask (vmode, 0, 0);

      if (mode == SFmode)
	copysign_insn = gen_copysignsf3_const;
      else if (mode == DFmode)
	copysign_insn = gen_copysigndf3_const;
      else
	copysign_insn = gen_copysigntf3_const;

      emit_insn (copysign_insn (dest, op0, op1, mask));
    }
  else
    {
      rtx (*copysign_insn)(rtx, rtx, rtx, rtx, rtx, rtx);

      nmask = ix86_build_signbit_mask (vmode, 0, 1);
      mask = ix86_build_signbit_mask (vmode, 0, 0);

      if (mode == SFmode)
	copysign_insn = gen_copysignsf3_var;
      else if (mode == DFmode)
	copysign_insn = gen_copysigndf3_var;
      else
	copysign_insn = gen_copysigntf3_var;

      emit_insn (copysign_insn (dest, NULL_RTX, op0, op1, nmask, mask));
    }
}

/* Deconstruct a copysign operation into bit masks.  Operand 0 is known to
   be a constant, and so has already been expanded into a vector constant.  */

void
ix86_split_copysign_const (rtx operands[])
{
  machine_mode mode, vmode;
  rtx dest, op0, mask, x;

  dest = operands[0];
  op0 = operands[1];
  mask = operands[3];

  mode = GET_MODE (dest);
  vmode = GET_MODE (mask);

  dest = lowpart_subreg (vmode, dest, mode);
  x = gen_rtx_AND (vmode, dest, mask);
  emit_insn (gen_rtx_SET (dest, x));

  if (op0 != CONST0_RTX (vmode))
    {
      x = gen_rtx_IOR (vmode, dest, op0);
      emit_insn (gen_rtx_SET (dest, x));
    }
}

/* Deconstruct a copysign operation into bit masks.  Operand 0 is variable,
   so we have to do two masks.  */

void
ix86_split_copysign_var (rtx operands[])
{
  machine_mode mode, vmode;
  rtx dest, scratch, op0, op1, mask, nmask, x;

  dest = operands[0];
  scratch = operands[1];
  op0 = operands[2];
  op1 = operands[3];
  nmask = operands[4];
  mask = operands[5];

  mode = GET_MODE (dest);
  vmode = GET_MODE (mask);

  if (rtx_equal_p (op0, op1))
    {
      /* Shouldn't happen often (it's useless, obviously), but when it does
	 we'd generate incorrect code if we continue below.  */
      emit_move_insn (dest, op0);
      return;
    }

  if (REG_P (mask) && REGNO (dest) == REGNO (mask))	/* alternative 0 */
    {
      gcc_assert (REGNO (op1) == REGNO (scratch));

      x = gen_rtx_AND (vmode, scratch, mask);
      emit_insn (gen_rtx_SET (scratch, x));

      dest = mask;
      op0 = lowpart_subreg (vmode, op0, mode);
      x = gen_rtx_NOT (vmode, dest);
      x = gen_rtx_AND (vmode, x, op0);
      emit_insn (gen_rtx_SET (dest, x));
    }
  else
    {
      if (REGNO (op1) == REGNO (scratch))		/* alternative 1,3 */
	{
	  x = gen_rtx_AND (vmode, scratch, mask);
	}
      else						/* alternative 2,4 */
	{
          gcc_assert (REGNO (mask) == REGNO (scratch));
          op1 = lowpart_subreg (vmode, op1, mode);
	  x = gen_rtx_AND (vmode, scratch, op1);
	}
      emit_insn (gen_rtx_SET (scratch, x));

      if (REGNO (op0) == REGNO (dest))			/* alternative 1,2 */
	{
	  dest = lowpart_subreg (vmode, op0, mode);
	  x = gen_rtx_AND (vmode, dest, nmask);
	}
      else						/* alternative 3,4 */
	{
          gcc_assert (REGNO (nmask) == REGNO (dest));
	  dest = nmask;
	  op0 = lowpart_subreg (vmode, op0, mode);
	  x = gen_rtx_AND (vmode, dest, op0);
	}
      emit_insn (gen_rtx_SET (dest, x));
    }

  x = gen_rtx_IOR (vmode, dest, scratch);
  emit_insn (gen_rtx_SET (dest, x));
}

/* Return TRUE or FALSE depending on whether the first SET in INSN
   has source and destination with matching CC modes, and that the
   CC mode is at least as constrained as REQ_MODE.  */

bool
ix86_match_ccmode (rtx insn, machine_mode req_mode)
{
  rtx set;
  machine_mode set_mode;

  set = PATTERN (insn);
  if (GET_CODE (set) == PARALLEL)
    set = XVECEXP (set, 0, 0);
  gcc_assert (GET_CODE (set) == SET);
  gcc_assert (GET_CODE (SET_SRC (set)) == COMPARE);

  set_mode = GET_MODE (SET_DEST (set));
  switch (set_mode)
    {
    case CCNOmode:
      if (req_mode != CCNOmode
	  && (req_mode != CCmode
	      || XEXP (SET_SRC (set), 1) != const0_rtx))
	return false;
      break;
    case CCmode:
      if (req_mode == CCGCmode)
	return false;
      /* FALLTHRU */
    case CCGCmode:
      if (req_mode == CCGOCmode || req_mode == CCNOmode)
	return false;
      /* FALLTHRU */
    case CCGOCmode:
      if (req_mode == CCZmode)
	return false;
      /* FALLTHRU */
    case CCZmode:
      break;

    case CCAmode:
    case CCCmode:
    case CCOmode:
    case CCPmode:
    case CCSmode:
      if (set_mode != req_mode)
	return false;
      break;

    default:
      gcc_unreachable ();
    }

  return GET_MODE (SET_SRC (set)) == set_mode;
}

/* Generate insn patterns to do an integer compare of OPERANDS.  */

static rtx
ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
{
  machine_mode cmpmode;
  rtx tmp, flags;

  cmpmode = SELECT_CC_MODE (code, op0, op1);
  flags = gen_rtx_REG (cmpmode, FLAGS_REG);

  /* This is very simple, but making the interface the same as in the
     FP case makes the rest of the code easier.  */
  tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
  emit_insn (gen_rtx_SET (flags, tmp));

  /* Return the test that should be put into the flags user, i.e.
     the bcc, scc, or cmov instruction.  */
  return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
}

/* Figure out whether to use ordered or unordered fp comparisons.
   Return the appropriate mode to use.  */

machine_mode
ix86_fp_compare_mode (enum rtx_code)
{
  /* ??? In order to make all comparisons reversible, we do all comparisons
     non-trapping when compiling for IEEE.  Once gcc is able to distinguish
     all forms trapping and nontrapping comparisons, we can make inequality
     comparisons trapping again, since it results in better code when using
     FCOM based compares.  */
  return TARGET_IEEE_FP ? CCFPUmode : CCFPmode;
}

machine_mode
ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
{
  machine_mode mode = GET_MODE (op0);

  if (SCALAR_FLOAT_MODE_P (mode))
    {
      gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
      return ix86_fp_compare_mode (code);
    }

  switch (code)
    {
      /* Only zero flag is needed.  */
    case EQ:			/* ZF=0 */
    case NE:			/* ZF!=0 */
      return CCZmode;
      /* Codes needing carry flag.  */
    case GEU:			/* CF=0 */
    case LTU:			/* CF=1 */
      /* Detect overflow checks.  They need just the carry flag.  */
      if (GET_CODE (op0) == PLUS
	  && (rtx_equal_p (op1, XEXP (op0, 0))
	      || rtx_equal_p (op1, XEXP (op0, 1))))
	return CCCmode;
      else
	return CCmode;
    case GTU:			/* CF=0 & ZF=0 */
    case LEU:			/* CF=1 | ZF=1 */
      return CCmode;
      /* Codes possibly doable only with sign flag when
         comparing against zero.  */
    case GE:			/* SF=OF   or   SF=0 */
    case LT:			/* SF<>OF  or   SF=1 */
      if (op1 == const0_rtx)
	return CCGOCmode;
      else
	/* For other cases Carry flag is not required.  */
	return CCGCmode;
      /* Codes doable only with sign flag when comparing
         against zero, but we miss jump instruction for it
         so we need to use relational tests against overflow
         that thus needs to be zero.  */
    case GT:			/* ZF=0 & SF=OF */
    case LE:			/* ZF=1 | SF<>OF */
      if (op1 == const0_rtx)
	return CCNOmode;
      else
	return CCGCmode;
      /* strcmp pattern do (use flags) and combine may ask us for proper
	 mode.  */
    case USE:
      return CCmode;
    default:
      gcc_unreachable ();
    }
}

/* Return the fixed registers used for condition codes.  */

static bool
ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2)
{
  *p1 = FLAGS_REG;
  *p2 = FPSR_REG;
  return true;
}

/* If two condition code modes are compatible, return a condition code
   mode which is compatible with both.  Otherwise, return
   VOIDmode.  */

static machine_mode
ix86_cc_modes_compatible (machine_mode m1, machine_mode m2)
{
  if (m1 == m2)
    return m1;

  if (GET_MODE_CLASS (m1) != MODE_CC || GET_MODE_CLASS (m2) != MODE_CC)
    return VOIDmode;

  if ((m1 == CCGCmode && m2 == CCGOCmode)
      || (m1 == CCGOCmode && m2 == CCGCmode))
    return CCGCmode;

  if (m1 == CCZmode && (m2 == CCGCmode || m2 == CCGOCmode))
    return m2;
  else if (m2 == CCZmode && (m1 == CCGCmode || m1 == CCGOCmode))
    return m1;

  switch (m1)
    {
    default:
      gcc_unreachable ();

    case CCmode:
    case CCGCmode:
    case CCGOCmode:
    case CCNOmode:
    case CCAmode:
    case CCCmode:
    case CCOmode:
    case CCPmode:
    case CCSmode:
    case CCZmode:
      switch (m2)
	{
	default:
	  return VOIDmode;

	case CCmode:
	case CCGCmode:
	case CCGOCmode:
	case CCNOmode:
	case CCAmode:
	case CCCmode:
	case CCOmode:
	case CCPmode:
	case CCSmode:
	case CCZmode:
	  return CCmode;
	}

    case CCFPmode:
    case CCFPUmode:
      /* These are only compatible with themselves, which we already
	 checked above.  */
      return VOIDmode;
    }
}


/* Return a comparison we can do and that it is equivalent to
   swap_condition (code) apart possibly from orderedness.
   But, never change orderedness if TARGET_IEEE_FP, returning
   UNKNOWN in that case if necessary.  */

static enum rtx_code
ix86_fp_swap_condition (enum rtx_code code)
{
  switch (code)
    {
    case GT:                   /* GTU - CF=0 & ZF=0 */
      return TARGET_IEEE_FP ? UNKNOWN : UNLT;
    case GE:                   /* GEU - CF=0 */
      return TARGET_IEEE_FP ? UNKNOWN : UNLE;
    case UNLT:                 /* LTU - CF=1 */
      return TARGET_IEEE_FP ? UNKNOWN : GT;
    case UNLE:                 /* LEU - CF=1 | ZF=1 */
      return TARGET_IEEE_FP ? UNKNOWN : GE;
    default:
      return swap_condition (code);
    }
}

/* Return cost of comparison CODE using the best strategy for performance.
   All following functions do use number of instructions as a cost metrics.
   In future this should be tweaked to compute bytes for optimize_size and
   take into account performance of various instructions on various CPUs.  */

static int
ix86_fp_comparison_cost (enum rtx_code code)
{
  int arith_cost;

  /* The cost of code using bit-twiddling on %ah.  */
  switch (code)
    {
    case UNLE:
    case UNLT:
    case LTGT:
    case GT:
    case GE:
    case UNORDERED:
    case ORDERED:
    case UNEQ:
      arith_cost = 4;
      break;
    case LT:
    case NE:
    case EQ:
    case UNGE:
      arith_cost = TARGET_IEEE_FP ? 5 : 4;
      break;
    case LE:
    case UNGT:
      arith_cost = TARGET_IEEE_FP ? 6 : 4;
      break;
    default:
      gcc_unreachable ();
    }

  switch (ix86_fp_comparison_strategy (code))
    {
    case IX86_FPCMP_COMI:
      return arith_cost > 4 ? 3 : 2;
    case IX86_FPCMP_SAHF:
      return arith_cost > 4 ? 4 : 3;
    default:
      return arith_cost;
    }
}

/* Return strategy to use for floating-point.  We assume that fcomi is always
   preferrable where available, since that is also true when looking at size
   (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test).  */

enum ix86_fpcmp_strategy
ix86_fp_comparison_strategy (enum rtx_code)
{
  /* Do fcomi/sahf based test when profitable.  */

  if (TARGET_CMOVE)
    return IX86_FPCMP_COMI;

  if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
    return IX86_FPCMP_SAHF;

  return IX86_FPCMP_ARITH;
}

/* Swap, force into registers, or otherwise massage the two operands
   to a fp comparison.  The operands are updated in place; the new
   comparison code is returned.  */

static enum rtx_code
ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
{
  machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
  rtx op0 = *pop0, op1 = *pop1;
  machine_mode op_mode = GET_MODE (op0);
  int is_sse = TARGET_SSE_MATH && SSE_FLOAT_MODE_P (op_mode);

  /* All of the unordered compare instructions only work on registers.
     The same is true of the fcomi compare instructions.  The XFmode
     compare instructions require registers except when comparing
     against zero or when converting operand 1 from fixed point to
     floating point.  */

  if (!is_sse
      && (fpcmp_mode == CCFPUmode
	  || (op_mode == XFmode
	      && ! (standard_80387_constant_p (op0) == 1
		    || standard_80387_constant_p (op1) == 1)
	      && GET_CODE (op1) != FLOAT)
	  || ix86_fp_comparison_strategy (code) == IX86_FPCMP_COMI))
    {
      op0 = force_reg (op_mode, op0);
      op1 = force_reg (op_mode, op1);
    }
  else
    {
      /* %%% We only allow op1 in memory; op0 must be st(0).  So swap
	 things around if they appear profitable, otherwise force op0
	 into a register.  */

      if (standard_80387_constant_p (op0) == 0
	  || (MEM_P (op0)
	      && ! (standard_80387_constant_p (op1) == 0
		    || MEM_P (op1))))
	{
	  enum rtx_code new_code = ix86_fp_swap_condition (code);
	  if (new_code != UNKNOWN)
	    {
	      std::swap (op0, op1);
	      code = new_code;
	    }
	}

      if (!REG_P (op0))
	op0 = force_reg (op_mode, op0);

      if (CONSTANT_P (op1))
	{
	  int tmp = standard_80387_constant_p (op1);
	  if (tmp == 0)
	    op1 = validize_mem (force_const_mem (op_mode, op1));
	  else if (tmp == 1)
	    {
	      if (TARGET_CMOVE)
		op1 = force_reg (op_mode, op1);
	    }
	  else
	    op1 = force_reg (op_mode, op1);
	}
    }

  /* Try to rearrange the comparison to make it cheaper.  */
  if (ix86_fp_comparison_cost (code)
      > ix86_fp_comparison_cost (swap_condition (code))
      && (REG_P (op1) || can_create_pseudo_p ()))
    {
      std::swap (op0, op1);
      code = swap_condition (code);
      if (!REG_P (op0))
	op0 = force_reg (op_mode, op0);
    }

  *pop0 = op0;
  *pop1 = op1;
  return code;
}

/* Convert comparison codes we use to represent FP comparison to integer
   code that will result in proper branch.  Return UNKNOWN if no such code
   is available.  */

enum rtx_code
ix86_fp_compare_code_to_integer (enum rtx_code code)
{
  switch (code)
    {
    case GT:
      return GTU;
    case GE:
      return GEU;
    case ORDERED:
    case UNORDERED:
      return code;
    case UNEQ:
      return EQ;
    case UNLT:
      return LTU;
    case UNLE:
      return LEU;
    case LTGT:
      return NE;
    default:
      return UNKNOWN;
    }
}

/* Generate insn patterns to do a floating point compare of OPERANDS.  */

static rtx
ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1, rtx scratch)
{
  machine_mode fpcmp_mode, intcmp_mode;
  rtx tmp, tmp2;

  fpcmp_mode = ix86_fp_compare_mode (code);
  code = ix86_prepare_fp_compare_args (code, &op0, &op1);

  /* Do fcomi/sahf based test when profitable.  */
  switch (ix86_fp_comparison_strategy (code))
    {
    case IX86_FPCMP_COMI:
      intcmp_mode = fpcmp_mode;
      tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
      tmp = gen_rtx_SET (gen_rtx_REG (fpcmp_mode, FLAGS_REG), tmp);
      emit_insn (tmp);
      break;

    case IX86_FPCMP_SAHF:
      intcmp_mode = fpcmp_mode;
      tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
      tmp = gen_rtx_SET (gen_rtx_REG (fpcmp_mode, FLAGS_REG), tmp);

      if (!scratch)
	scratch = gen_reg_rtx (HImode);
      tmp2 = gen_rtx_CLOBBER (VOIDmode, scratch);
      emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, tmp2)));
      break;

    case IX86_FPCMP_ARITH:
      /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first.  */
      tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
      tmp2 = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
      if (!scratch)
	scratch = gen_reg_rtx (HImode);
      emit_insn (gen_rtx_SET (scratch, tmp2));

      /* In the unordered case, we have to check C2 for NaN's, which
	 doesn't happen to work out to anything nice combination-wise.
	 So do some bit twiddling on the value we've got in AH to come
	 up with an appropriate set of condition codes.  */

      intcmp_mode = CCNOmode;
      switch (code)
	{
	case GT:
	case UNGT:
	  if (code == GT || !TARGET_IEEE_FP)
	    {
	      emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
	      code = EQ;
	    }
	  else
	    {
	      emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
	      emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
	      emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
	      intcmp_mode = CCmode;
	      code = GEU;
	    }
	  break;
	case LT:
	case UNLT:
	  if (code == LT && TARGET_IEEE_FP)
	    {
	      emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
	      emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx));
	      intcmp_mode = CCmode;
	      code = EQ;
	    }
	  else
	    {
	      emit_insn (gen_testqi_ext_1_ccno (scratch, const1_rtx));
	      code = NE;
	    }
	  break;
	case GE:
	case UNGE:
	  if (code == GE || !TARGET_IEEE_FP)
	    {
	      emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x05)));
	      code = EQ;
	    }
	  else
	    {
	      emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
	      emit_insn (gen_xorqi_ext_1_cc (scratch, scratch, const1_rtx));
	      code = NE;
	    }
	  break;
	case LE:
	case UNLE:
	  if (code == LE && TARGET_IEEE_FP)
	    {
	      emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
	      emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
	      emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
	      intcmp_mode = CCmode;
	      code = LTU;
	    }
	  else
	    {
	      emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
	      code = NE;
	    }
	  break;
	case EQ:
	case UNEQ:
	  if (code == EQ && TARGET_IEEE_FP)
	    {
	      emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
	      emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
	      intcmp_mode = CCmode;
	      code = EQ;
	    }
	  else
	    {
	      emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
	      code = NE;
	    }
	  break;
	case NE:
	case LTGT:
	  if (code == NE && TARGET_IEEE_FP)
	    {
	      emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
	      emit_insn (gen_xorqi_ext_1_cc (scratch, scratch,
					     GEN_INT (0x40)));
	      code = NE;
	    }
	  else
	    {
	      emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
	      code = EQ;
	    }
	  break;

	case UNORDERED:
	  emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
	  code = NE;
	  break;
	case ORDERED:
	  emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
	  code = EQ;
	  break;

	default:
	  gcc_unreachable ();
	}
	break;

    default:
      gcc_unreachable();
    }

  /* Return the test that should be put into the flags user, i.e.
     the bcc, scc, or cmov instruction.  */
  return gen_rtx_fmt_ee (code, VOIDmode,
			 gen_rtx_REG (intcmp_mode, FLAGS_REG),
			 const0_rtx);
}

static rtx
ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1)
{
  rtx ret;

  if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
    ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1);

  else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
    {
      gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
      ret = ix86_expand_fp_compare (code, op0, op1, NULL_RTX);
    }
  else
    ret = ix86_expand_int_compare (code, op0, op1);

  return ret;
}

void
ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
{
  machine_mode mode = GET_MODE (op0);
  rtx tmp;

  /* Handle special case - vector comparsion with boolean result, transform
     it using ptest instruction.  */
  if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
    {
      rtx flag = gen_rtx_REG (CCZmode, FLAGS_REG);
      machine_mode p_mode = GET_MODE_SIZE (mode) == 32 ? V4DImode : V2DImode;

      gcc_assert (code == EQ || code == NE);
      /* Generate XOR since we can't check that one operand is zero vector.  */
      tmp = gen_reg_rtx (mode);
      emit_insn (gen_rtx_SET (tmp, gen_rtx_XOR (mode, op0, op1)));
      tmp = gen_lowpart (p_mode, tmp);
      emit_insn (gen_rtx_SET (gen_rtx_REG (CCmode, FLAGS_REG),
			      gen_rtx_UNSPEC (CCmode,
					      gen_rtvec (2, tmp, tmp),
					      UNSPEC_PTEST)));
      tmp = gen_rtx_fmt_ee (code, VOIDmode, flag, const0_rtx);
      tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
				  gen_rtx_LABEL_REF (VOIDmode, label),
				  pc_rtx);
      emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
      return;
    }

  switch (mode)
    {
    case SFmode:
    case DFmode:
    case XFmode:
    case QImode:
    case HImode:
    case SImode:
      simple:
      tmp = ix86_expand_compare (code, op0, op1);
      tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
				  gen_rtx_LABEL_REF (VOIDmode, label),
				  pc_rtx);
      emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
      return;

    case DImode:
      if (TARGET_64BIT)
	goto simple;
      /* For 32-bit target DI comparison may be performed on
	 SSE registers.  To allow this we should avoid split
	 to SI mode which is achieved by doing xor in DI mode
	 and then comparing with zero (which is recognized by
	 STV pass).  We don't compare using xor when optimizing
	 for size.  */
      if (!optimize_insn_for_size_p ()
	  && TARGET_STV
	  && (code == EQ || code == NE))
	{
	  op0 = force_reg (mode, gen_rtx_XOR (mode, op0, op1));
	  op1 = const0_rtx;
	}
      /* FALLTHRU */
    case TImode:
      /* Expand DImode branch into multiple compare+branch.  */
      {
	rtx lo[2], hi[2];
	rtx_code_label *label2;
	enum rtx_code code1, code2, code3;
	machine_mode submode;

	if (CONSTANT_P (op0) && !CONSTANT_P (op1))
	  {
	    std::swap (op0, op1);
	    code = swap_condition (code);
	  }

	split_double_mode (mode, &op0, 1, lo+0, hi+0);
	split_double_mode (mode, &op1, 1, lo+1, hi+1);

	submode = mode == DImode ? SImode : DImode;

	/* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
	   avoid two branches.  This costs one extra insn, so disable when
	   optimizing for size.  */

	if ((code == EQ || code == NE)
	    && (!optimize_insn_for_size_p ()
	        || hi[1] == const0_rtx || lo[1] == const0_rtx))
	  {
	    rtx xor0, xor1;

	    xor1 = hi[0];
	    if (hi[1] != const0_rtx)
	      xor1 = expand_binop (submode, xor_optab, xor1, hi[1],
				   NULL_RTX, 0, OPTAB_WIDEN);

	    xor0 = lo[0];
	    if (lo[1] != const0_rtx)
	      xor0 = expand_binop (submode, xor_optab, xor0, lo[1],
				   NULL_RTX, 0, OPTAB_WIDEN);

	    tmp = expand_binop (submode, ior_optab, xor1, xor0,
				NULL_RTX, 0, OPTAB_WIDEN);

	    ix86_expand_branch (code, tmp, const0_rtx, label);
	    return;
	  }

	/* Otherwise, if we are doing less-than or greater-or-equal-than,
	   op1 is a constant and the low word is zero, then we can just
	   examine the high word.  Similarly for low word -1 and
	   less-or-equal-than or greater-than.  */

	if (CONST_INT_P (hi[1]))
	  switch (code)
	    {
	    case LT: case LTU: case GE: case GEU:
	      if (lo[1] == const0_rtx)
		{
		  ix86_expand_branch (code, hi[0], hi[1], label);
		  return;
		}
	      break;
	    case LE: case LEU: case GT: case GTU:
	      if (lo[1] == constm1_rtx)
		{
		  ix86_expand_branch (code, hi[0], hi[1], label);
		  return;
		}
	      break;
	    default:
	      break;
	    }

	/* Otherwise, we need two or three jumps.  */

	label2 = gen_label_rtx ();

	code1 = code;
	code2 = swap_condition (code);
	code3 = unsigned_condition (code);

	switch (code)
	  {
	  case LT: case GT: case LTU: case GTU:
	    break;

	  case LE:   code1 = LT;  code2 = GT;  break;
	  case GE:   code1 = GT;  code2 = LT;  break;
	  case LEU:  code1 = LTU; code2 = GTU; break;
	  case GEU:  code1 = GTU; code2 = LTU; break;

	  case EQ:   code1 = UNKNOWN; code2 = NE;  break;
	  case NE:   code2 = UNKNOWN; break;

	  default:
	    gcc_unreachable ();
	  }

	/*
	 * a < b =>
	 *    if (hi(a) < hi(b)) goto true;
	 *    if (hi(a) > hi(b)) goto false;
	 *    if (lo(a) < lo(b)) goto true;
	 *  false:
	 */

	if (code1 != UNKNOWN)
	  ix86_expand_branch (code1, hi[0], hi[1], label);
	if (code2 != UNKNOWN)
	  ix86_expand_branch (code2, hi[0], hi[1], label2);

	ix86_expand_branch (code3, lo[0], lo[1], label);

	if (code2 != UNKNOWN)
	  emit_label (label2);
	return;
      }

    default:
      gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC);
      goto simple;
    }
}

/* Split branch based on floating point condition.  */
void
ix86_split_fp_branch (enum rtx_code code, rtx op1, rtx op2,
		      rtx target1, rtx target2, rtx tmp)
{
  rtx condition;
  rtx_insn *i;

  if (target2 != pc_rtx)
    {
      std::swap (target1, target2);
      code = reverse_condition_maybe_unordered (code);
    }

  condition = ix86_expand_fp_compare (code, op1, op2,
				      tmp);

  i = emit_jump_insn (gen_rtx_SET
		      (pc_rtx,
		       gen_rtx_IF_THEN_ELSE (VOIDmode,
					     condition, target1, target2)));
  if (split_branch_probability >= 0)
    add_int_reg_note (i, REG_BR_PROB, split_branch_probability);
}

void
ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1)
{
  rtx ret;

  gcc_assert (GET_MODE (dest) == QImode);

  ret = ix86_expand_compare (code, op0, op1);
  PUT_MODE (ret, QImode);
  emit_insn (gen_rtx_SET (dest, ret));
}

/* Expand comparison setting or clearing carry flag.  Return true when
   successful and set pop for the operation.  */
static bool
ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
{
  machine_mode mode =
    GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);

  /* Do not handle double-mode compares that go through special path.  */
  if (mode == (TARGET_64BIT ? TImode : DImode))
    return false;

  if (SCALAR_FLOAT_MODE_P (mode))
    {
      rtx compare_op;
      rtx_insn *compare_seq;

      gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));

      /* Shortcut:  following common codes never translate
	 into carry flag compares.  */
      if (code == EQ || code == NE || code == UNEQ || code == LTGT
	  || code == ORDERED || code == UNORDERED)
	return false;

      /* These comparisons require zero flag; swap operands so they won't.  */
      if ((code == GT || code == UNLE || code == LE || code == UNGT)
	  && !TARGET_IEEE_FP)
	{
	  std::swap (op0, op1);
	  code = swap_condition (code);
	}

      /* Try to expand the comparison and verify that we end up with
	 carry flag based comparison.  This fails to be true only when
	 we decide to expand comparison using arithmetic that is not
	 too common scenario.  */
      start_sequence ();
      compare_op = ix86_expand_fp_compare (code, op0, op1, NULL_RTX);
      compare_seq = get_insns ();
      end_sequence ();

      if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
	  || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
        code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
      else
	code = GET_CODE (compare_op);

      if (code != LTU && code != GEU)
	return false;

      emit_insn (compare_seq);
      *pop = compare_op;
      return true;
    }

  if (!INTEGRAL_MODE_P (mode))
    return false;

  switch (code)
    {
    case LTU:
    case GEU:
      break;

    /* Convert a==0 into (unsigned)a<1.  */
    case EQ:
    case NE:
      if (op1 != const0_rtx)
	return false;
      op1 = const1_rtx;
      code = (code == EQ ? LTU : GEU);
      break;

    /* Convert a>b into b<a or a>=b-1.  */
    case GTU:
    case LEU:
      if (CONST_INT_P (op1))
	{
	  op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
	  /* Bail out on overflow.  We still can swap operands but that
	     would force loading of the constant into register.  */
	  if (op1 == const0_rtx
	      || !x86_64_immediate_operand (op1, GET_MODE (op1)))
	    return false;
	  code = (code == GTU ? GEU : LTU);
	}
      else
	{
	  std::swap (op0, op1);
	  code = (code == GTU ? LTU : GEU);
	}
      break;

    /* Convert a>=0 into (unsigned)a<0x80000000.  */
    case LT:
    case GE:
      if (mode == DImode || op1 != const0_rtx)
	return false;
      op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
      code = (code == LT ? GEU : LTU);
      break;
    case LE:
    case GT:
      if (mode == DImode || op1 != constm1_rtx)
	return false;
      op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
      code = (code == LE ? GEU : LTU);
      break;

    default:
      return false;
    }
  /* Swapping operands may cause constant to appear as first operand.  */
  if (!nonimmediate_operand (op0, VOIDmode))
    {
      if (!can_create_pseudo_p ())
	return false;
      op0 = force_reg (mode, op0);
    }
  *pop = ix86_expand_compare (code, op0, op1);
  gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
  return true;
}

bool
ix86_expand_int_movcc (rtx operands[])
{
  enum rtx_code code = GET_CODE (operands[1]), compare_code;
  rtx_insn *compare_seq;
  rtx compare_op;
  machine_mode mode = GET_MODE (operands[0]);
  bool sign_bit_compare_p = false;
  rtx op0 = XEXP (operands[1], 0);
  rtx op1 = XEXP (operands[1], 1);

  if (GET_MODE (op0) == TImode
      || (GET_MODE (op0) == DImode
	  && !TARGET_64BIT))
    return false;

  start_sequence ();
  compare_op = ix86_expand_compare (code, op0, op1);
  compare_seq = get_insns ();
  end_sequence ();

  compare_code = GET_CODE (compare_op);

  if ((op1 == const0_rtx && (code == GE || code == LT))
      || (op1 == constm1_rtx && (code == GT || code == LE)))
    sign_bit_compare_p = true;

  /* Don't attempt mode expansion here -- if we had to expand 5 or 6
     HImode insns, we'd be swallowed in word prefix ops.  */

  if ((mode != HImode || TARGET_FAST_PREFIX)
      && (mode != (TARGET_64BIT ? TImode : DImode))
      && CONST_INT_P (operands[2])
      && CONST_INT_P (operands[3]))
    {
      rtx out = operands[0];
      HOST_WIDE_INT ct = INTVAL (operands[2]);
      HOST_WIDE_INT cf = INTVAL (operands[3]);
      HOST_WIDE_INT diff;

      diff = ct - cf;
      /*  Sign bit compares are better done using shifts than we do by using
	  sbb.  */
      if (sign_bit_compare_p
	  || ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
	{
	  /* Detect overlap between destination and compare sources.  */
	  rtx tmp = out;

          if (!sign_bit_compare_p)
	    {
	      rtx flags;
	      bool fpcmp = false;

	      compare_code = GET_CODE (compare_op);

	      flags = XEXP (compare_op, 0);

	      if (GET_MODE (flags) == CCFPmode
		  || GET_MODE (flags) == CCFPUmode)
		{
		  fpcmp = true;
		  compare_code
		    = ix86_fp_compare_code_to_integer (compare_code);
		}

	      /* To simplify rest of code, restrict to the GEU case.  */
	      if (compare_code == LTU)
		{
		  std::swap (ct, cf);
		  compare_code = reverse_condition (compare_code);
		  code = reverse_condition (code);
		}
	      else
		{
		  if (fpcmp)
		    PUT_CODE (compare_op,
			      reverse_condition_maybe_unordered
			        (GET_CODE (compare_op)));
		  else
		    PUT_CODE (compare_op,
			      reverse_condition (GET_CODE (compare_op)));
		}
	      diff = ct - cf;

	      if (reg_overlap_mentioned_p (out, op0)
		  || reg_overlap_mentioned_p (out, op1))
		tmp = gen_reg_rtx (mode);

	      if (mode == DImode)
		emit_insn (gen_x86_movdicc_0_m1 (tmp, flags, compare_op));
	      else
		emit_insn (gen_x86_movsicc_0_m1	(gen_lowpart (SImode, tmp),
						 flags, compare_op));
	    }
	  else
	    {
	      if (code == GT || code == GE)
		code = reverse_condition (code);
	      else
		{
		  std::swap (ct, cf);
		  diff = ct - cf;
		}
	      tmp = emit_store_flag (tmp, code, op0, op1, VOIDmode, 0, -1);
	    }

	  if (diff == 1)
	    {
	      /*
	       * cmpl op0,op1
	       * sbbl dest,dest
	       * [addl dest, ct]
	       *
	       * Size 5 - 8.
	       */
	      if (ct)
		tmp = expand_simple_binop (mode, PLUS,
					   tmp, GEN_INT (ct),
					   copy_rtx (tmp), 1, OPTAB_DIRECT);
	    }
	  else if (cf == -1)
	    {
	      /*
	       * cmpl op0,op1
	       * sbbl dest,dest
	       * orl $ct, dest
	       *
	       * Size 8.
	       */
	      tmp = expand_simple_binop (mode, IOR,
					 tmp, GEN_INT (ct),
					 copy_rtx (tmp), 1, OPTAB_DIRECT);
	    }
	  else if (diff == -1 && ct)
	    {
	      /*
	       * cmpl op0,op1
	       * sbbl dest,dest
	       * notl dest
	       * [addl dest, cf]
	       *
	       * Size 8 - 11.
	       */
	      tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
	      if (cf)
		tmp = expand_simple_binop (mode, PLUS,
					   copy_rtx (tmp), GEN_INT (cf),
					   copy_rtx (tmp), 1, OPTAB_DIRECT);
	    }
	  else
	    {
	      /*
	       * cmpl op0,op1
	       * sbbl dest,dest
	       * [notl dest]
	       * andl cf - ct, dest
	       * [addl dest, ct]
	       *
	       * Size 8 - 11.
	       */

	      if (cf == 0)
		{
		  cf = ct;
		  ct = 0;
		  tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
		}

	      tmp = expand_simple_binop (mode, AND,
					 copy_rtx (tmp),
					 gen_int_mode (cf - ct, mode),
					 copy_rtx (tmp), 1, OPTAB_DIRECT);
	      if (ct)
		tmp = expand_simple_binop (mode, PLUS,
					   copy_rtx (tmp), GEN_INT (ct),
					   copy_rtx (tmp), 1, OPTAB_DIRECT);
	    }

	  if (!rtx_equal_p (tmp, out))
	    emit_move_insn (copy_rtx (out), copy_rtx (tmp));

	  return true;
	}

      if (diff < 0)
	{
	  machine_mode cmp_mode = GET_MODE (op0);
	  enum rtx_code new_code;

	  if (SCALAR_FLOAT_MODE_P (cmp_mode))
	    {
	      gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));

	      /* We may be reversing unordered compare to normal compare, that
		 is not valid in general (we may convert non-trapping condition
		 to trapping one), however on i386 we currently emit all
		 comparisons unordered.  */
	      new_code = reverse_condition_maybe_unordered (code);
	    }
	  else
	    new_code = ix86_reverse_condition (code, cmp_mode);
	  if (new_code != UNKNOWN)
	    {
	      std::swap (ct, cf);
	      diff = -diff;
	      code = new_code;
	    }
	}

      compare_code = UNKNOWN;
      if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_INT
	  && CONST_INT_P (op1))
	{
	  if (op1 == const0_rtx
	      && (code == LT || code == GE))
	    compare_code = code;
	  else if (op1 == constm1_rtx)
	    {
	      if (code == LE)
		compare_code = LT;
	      else if (code == GT)
		compare_code = GE;
	    }
	}

      /* Optimize dest = (op0 < 0) ? -1 : cf.  */
      if (compare_code != UNKNOWN
	  && GET_MODE (op0) == GET_MODE (out)
	  && (cf == -1 || ct == -1))
	{
	  /* If lea code below could be used, only optimize
	     if it results in a 2 insn sequence.  */

	  if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
		 || diff == 3 || diff == 5 || diff == 9)
	      || (compare_code == LT && ct == -1)
	      || (compare_code == GE && cf == -1))
	    {
	      /*
	       * notl op1	(if necessary)
	       * sarl $31, op1
	       * orl cf, op1
	       */
	      if (ct != -1)
		{
		  cf = ct;
		  ct = -1;
		  code = reverse_condition (code);
		}

	      out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);

	      out = expand_simple_binop (mode, IOR,
					 out, GEN_INT (cf),
					 out, 1, OPTAB_DIRECT);
	      if (out != operands[0])
		emit_move_insn (operands[0], out);

	      return true;
	    }
	}


      if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
	   || diff == 3 || diff == 5 || diff == 9)
	  && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
	  && (mode != DImode
	      || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
	{
	  /*
	   * xorl dest,dest
	   * cmpl op1,op2
	   * setcc dest
	   * lea cf(dest*(ct-cf)),dest
	   *
	   * Size 14.
	   *
	   * This also catches the degenerate setcc-only case.
	   */

	  rtx tmp;
	  int nops;

	  out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);

	  nops = 0;
	  /* On x86_64 the lea instruction operates on Pmode, so we need
	     to get arithmetics done in proper mode to match.  */
	  if (diff == 1)
	    tmp = copy_rtx (out);
	  else
	    {
	      rtx out1;
	      out1 = copy_rtx (out);
	      tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
	      nops++;
	      if (diff & 1)
		{
		  tmp = gen_rtx_PLUS (mode, tmp, out1);
		  nops++;
		}
	    }
	  if (cf != 0)
	    {
	      tmp = gen_rtx_PLUS (mode, tmp, GEN_INT (cf));
	      nops++;
	    }
	  if (!rtx_equal_p (tmp, out))
	    {
	      if (nops == 1)
		out = force_operand (tmp, copy_rtx (out));
	      else
		emit_insn (gen_rtx_SET (copy_rtx (out), copy_rtx (tmp)));
	    }
	  if (!rtx_equal_p (out, operands[0]))
	    emit_move_insn (operands[0], copy_rtx (out));

	  return true;
	}

      /*
       * General case:			Jumpful:
       *   xorl dest,dest		cmpl op1, op2
       *   cmpl op1, op2		movl ct, dest
       *   setcc dest			jcc 1f
       *   decl dest			movl cf, dest
       *   andl (cf-ct),dest		1:
       *   addl ct,dest
       *
       * Size 20.			Size 14.
       *
       * This is reasonably steep, but branch mispredict costs are
       * high on modern cpus, so consider failing only if optimizing
       * for space.
       */

      if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
	  && BRANCH_COST (optimize_insn_for_speed_p (),
		  	  false) >= 2)
	{
	  if (cf == 0)
	    {
	      machine_mode cmp_mode = GET_MODE (op0);
	      enum rtx_code new_code;

	      if (SCALAR_FLOAT_MODE_P (cmp_mode))
		{
		  gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));

		  /* We may be reversing unordered compare to normal compare,
		     that is not valid in general (we may convert non-trapping
		     condition to trapping one), however on i386 we currently
		     emit all comparisons unordered.  */
		  new_code = reverse_condition_maybe_unordered (code);
		}
	      else
		{
		  new_code = ix86_reverse_condition (code, cmp_mode);
		  if (compare_code != UNKNOWN && new_code != UNKNOWN)
		    compare_code = reverse_condition (compare_code);
		}

	      if (new_code != UNKNOWN)
		{
		  cf = ct;
		  ct = 0;
		  code = new_code;
		}
	    }

	  if (compare_code != UNKNOWN)
	    {
	      /* notl op1	(if needed)
		 sarl $31, op1
		 andl (cf-ct), op1
		 addl ct, op1

		 For x < 0 (resp. x <= -1) there will be no notl,
		 so if possible swap the constants to get rid of the
		 complement.
		 True/false will be -1/0 while code below (store flag
		 followed by decrement) is 0/-1, so the constants need
		 to be exchanged once more.  */

	      if (compare_code == GE || !cf)
		{
		  code = reverse_condition (code);
		  compare_code = LT;
		}
	      else
		std::swap (ct, cf);

	      out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
	    }
	  else
	    {
	      out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);

	      out = expand_simple_binop (mode, PLUS, copy_rtx (out),
					 constm1_rtx,
					 copy_rtx (out), 1, OPTAB_DIRECT);
	    }

	  out = expand_simple_binop (mode, AND, copy_rtx (out),
				     gen_int_mode (cf - ct, mode),
				     copy_rtx (out), 1, OPTAB_DIRECT);
	  if (ct)
	    out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
				       copy_rtx (out), 1, OPTAB_DIRECT);
	  if (!rtx_equal_p (out, operands[0]))
	    emit_move_insn (operands[0], copy_rtx (out));

	  return true;
	}
    }

  if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
    {
      /* Try a few things more with specific constants and a variable.  */

      optab op;
      rtx var, orig_out, out, tmp;

      if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
	return false;

      /* If one of the two operands is an interesting constant, load a
	 constant with the above and mask it in with a logical operation.  */

      if (CONST_INT_P (operands[2]))
	{
	  var = operands[3];
	  if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
	    operands[3] = constm1_rtx, op = and_optab;
	  else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
	    operands[3] = const0_rtx, op = ior_optab;
	  else
	    return false;
	}
      else if (CONST_INT_P (operands[3]))
	{
	  var = operands[2];
	  if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
	    operands[2] = constm1_rtx, op = and_optab;
	  else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
	    operands[2] = const0_rtx, op = ior_optab;
	  else
	    return false;
	}
      else
        return false;

      orig_out = operands[0];
      tmp = gen_reg_rtx (mode);
      operands[0] = tmp;

      /* Recurse to get the constant loaded.  */
      if (!ix86_expand_int_movcc (operands))
        return false;

      /* Mask in the interesting variable.  */
      out = expand_binop (mode, op, var, tmp, orig_out, 0,
			  OPTAB_WIDEN);
      if (!rtx_equal_p (out, orig_out))
	emit_move_insn (copy_rtx (orig_out), copy_rtx (out));

      return true;
    }

  /*
   * For comparison with above,
   *
   * movl cf,dest
   * movl ct,tmp
   * cmpl op1,op2
   * cmovcc tmp,dest
   *
   * Size 15.
   */

  if (! nonimmediate_operand (operands[2], mode))
    operands[2] = force_reg (mode, operands[2]);
  if (! nonimmediate_operand (operands[3], mode))
    operands[3] = force_reg (mode, operands[3]);

  if (! register_operand (operands[2], VOIDmode)
      && (mode == QImode
          || ! register_operand (operands[3], VOIDmode)))
    operands[2] = force_reg (mode, operands[2]);

  if (mode == QImode
      && ! register_operand (operands[3], VOIDmode))
    operands[3] = force_reg (mode, operands[3]);

  emit_insn (compare_seq);
  emit_insn (gen_rtx_SET (operands[0],
			  gen_rtx_IF_THEN_ELSE (mode,
						compare_op, operands[2],
						operands[3])));
  return true;
}

/* Swap, force into registers, or otherwise massage the two operands
   to an sse comparison with a mask result.  Thus we differ a bit from
   ix86_prepare_fp_compare_args which expects to produce a flags result.

   The DEST operand exists to help determine whether to commute commutative
   operators.  The POP0/POP1 operands are updated in place.  The new
   comparison code is returned, or UNKNOWN if not implementable.  */

static enum rtx_code
ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
				  rtx *pop0, rtx *pop1)
{
  switch (code)
    {
    case LTGT:
    case UNEQ:
      /* AVX supports all the needed comparisons.  */
      if (TARGET_AVX)
	break;
      /* We have no LTGT as an operator.  We could implement it with
	 NE & ORDERED, but this requires an extra temporary.  It's
	 not clear that it's worth it.  */
      return UNKNOWN;

    case LT:
    case LE:
    case UNGT:
    case UNGE:
      /* These are supported directly.  */
      break;

    case EQ:
    case NE:
    case UNORDERED:
    case ORDERED:
      /* AVX has 3 operand comparisons, no need to swap anything.  */
      if (TARGET_AVX)
	break;
      /* For commutative operators, try to canonicalize the destination
	 operand to be first in the comparison - this helps reload to
	 avoid extra moves.  */
      if (!dest || !rtx_equal_p (dest, *pop1))
	break;
      /* FALLTHRU */

    case GE:
    case GT:
    case UNLE:
    case UNLT:
      /* These are not supported directly before AVX, and furthermore
	 ix86_expand_sse_fp_minmax only optimizes LT/UNGE.  Swap the
	 comparison operands to transform into something that is
	 supported.  */
      std::swap (*pop0, *pop1);
      code = swap_condition (code);
      break;

    default:
      gcc_unreachable ();
    }

  return code;
}

/* Detect conditional moves that exactly match min/max operational
   semantics.  Note that this is IEEE safe, as long as we don't
   interchange the operands.

   Returns FALSE if this conditional move doesn't match a MIN/MAX,
   and TRUE if the operation is successful and instructions are emitted.  */

static bool
ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
			   rtx cmp_op1, rtx if_true, rtx if_false)
{
  machine_mode mode;
  bool is_min;
  rtx tmp;

  if (code == LT)
    ;
  else if (code == UNGE)
    std::swap (if_true, if_false);
  else
    return false;

  if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
    is_min = true;
  else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
    is_min = false;
  else
    return false;

  mode = GET_MODE (dest);

  /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
     but MODE may be a vector mode and thus not appropriate.  */
  if (!flag_finite_math_only || flag_signed_zeros)
    {
      int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
      rtvec v;

      if_true = force_reg (mode, if_true);
      v = gen_rtvec (2, if_true, if_false);
      tmp = gen_rtx_UNSPEC (mode, v, u);
    }
  else
    {
      code = is_min ? SMIN : SMAX;
      tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
    }

  emit_insn (gen_rtx_SET (dest, tmp));
  return true;
}

/* Expand an sse vector comparison.  Return the register with the result.  */

static rtx
ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
		     rtx op_true, rtx op_false)
{
  machine_mode mode = GET_MODE (dest);
  machine_mode cmp_ops_mode = GET_MODE (cmp_op0);

  /* In general case result of comparison can differ from operands' type.  */
  machine_mode cmp_mode;

  /* In AVX512F the result of comparison is an integer mask.  */
  bool maskcmp = false;
  rtx x;

  if (GET_MODE_SIZE (cmp_ops_mode) == 64)
    {
      cmp_mode = mode_for_size (GET_MODE_NUNITS (cmp_ops_mode), MODE_INT, 0);
      gcc_assert (cmp_mode != BLKmode);

      maskcmp = true;
    }
  else
    cmp_mode = cmp_ops_mode;


  cmp_op0 = force_reg (cmp_ops_mode, cmp_op0);
  if (!nonimmediate_operand (cmp_op1, cmp_ops_mode))
    cmp_op1 = force_reg (cmp_ops_mode, cmp_op1);

  if (optimize
      || (maskcmp && cmp_mode != mode)
      || (op_true && reg_overlap_mentioned_p (dest, op_true))
      || (op_false && reg_overlap_mentioned_p (dest, op_false)))
    dest = gen_reg_rtx (maskcmp ? cmp_mode : mode);

  /* Compare patterns for int modes are unspec in AVX512F only.  */
  if (maskcmp && (code == GT || code == EQ))
    {
      rtx (*gen)(rtx, rtx, rtx);

      switch (cmp_ops_mode)
	{
	case V64QImode:
	  gcc_assert (TARGET_AVX512BW);
	  gen = code == GT ? gen_avx512bw_gtv64qi3 : gen_avx512bw_eqv64qi3_1;
	  break;
	case V32HImode:
	  gcc_assert (TARGET_AVX512BW);
	  gen = code == GT ? gen_avx512bw_gtv32hi3 : gen_avx512bw_eqv32hi3_1;
	  break;
	case V16SImode:
	  gen = code == GT ? gen_avx512f_gtv16si3 : gen_avx512f_eqv16si3_1;
	  break;
	case V8DImode:
	  gen = code == GT ? gen_avx512f_gtv8di3 : gen_avx512f_eqv8di3_1;
	  break;
	default:
	  gen = NULL;
	}

      if (gen)
	{
	  emit_insn (gen (dest, cmp_op0, cmp_op1));
	  return dest;
	}
    }
  x = gen_rtx_fmt_ee (code, cmp_mode, cmp_op0, cmp_op1);

  if (cmp_mode != mode && !maskcmp)
    {
      x = force_reg (cmp_ops_mode, x);
      convert_move (dest, x, false);
    }
  else
    emit_insn (gen_rtx_SET (dest, x));

  return dest;
}

/* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
   operations.  This is used for both scalar and vector conditional moves.  */

void
ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
{
  machine_mode mode = GET_MODE (dest);
  machine_mode cmpmode = GET_MODE (cmp);

  /* In AVX512F the result of comparison is an integer mask.  */
  bool maskcmp = (mode != cmpmode && TARGET_AVX512F);

  rtx t2, t3, x;

  /* If we have an integer mask and FP value then we need
     to cast mask to FP mode.  */
  if (mode != cmpmode && VECTOR_MODE_P (cmpmode))
    {
      cmp = force_reg (cmpmode, cmp);
      cmp = gen_rtx_SUBREG (mode, cmp, 0);
    }

  if (vector_all_ones_operand (op_true, mode)
      && rtx_equal_p (op_false, CONST0_RTX (mode))
      && !maskcmp)
    {
      emit_insn (gen_rtx_SET (dest, cmp));
    }
  else if (op_false == CONST0_RTX (mode)
      && !maskcmp)
    {
      op_true = force_reg (mode, op_true);
      x = gen_rtx_AND (mode, cmp, op_true);
      emit_insn (gen_rtx_SET (dest, x));
    }
  else if (op_true == CONST0_RTX (mode)
      && !maskcmp)
    {
      op_false = force_reg (mode, op_false);
      x = gen_rtx_NOT (mode, cmp);
      x = gen_rtx_AND (mode, x, op_false);
      emit_insn (gen_rtx_SET (dest, x));
    }
  else if (INTEGRAL_MODE_P (mode) && op_true == CONSTM1_RTX (mode)
      && !maskcmp)
    {
      op_false = force_reg (mode, op_false);
      x = gen_rtx_IOR (mode, cmp, op_false);
      emit_insn (gen_rtx_SET (dest, x));
    }
  else if (TARGET_XOP
      && !maskcmp)
    {
      op_true = force_reg (mode, op_true);

      if (!nonimmediate_operand (op_false, mode))
	op_false = force_reg (mode, op_false);

      emit_insn (gen_rtx_SET (dest, gen_rtx_IF_THEN_ELSE (mode, cmp,
							  op_true,
							  op_false)));
    }
  else
    {
      rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
      rtx d = dest;

      if (!nonimmediate_operand (op_true, mode))
	op_true = force_reg (mode, op_true);

      op_false = force_reg (mode, op_false);

      switch (mode)
	{
	case V4SFmode:
	  if (TARGET_SSE4_1)
	    gen = gen_sse4_1_blendvps;
	  break;
	case V2DFmode:
	  if (TARGET_SSE4_1)
	    gen = gen_sse4_1_blendvpd;
	  break;
	case V16QImode:
	case V8HImode:
	case V4SImode:
	case V2DImode:
	  if (TARGET_SSE4_1)
	    {
	      gen = gen_sse4_1_pblendvb;
	      if (mode != V16QImode)
		d = gen_reg_rtx (V16QImode);
	      op_false = gen_lowpart (V16QImode, op_false);
	      op_true = gen_lowpart (V16QImode, op_true);
	      cmp = gen_lowpart (V16QImode, cmp);
	    }
	  break;
	case V8SFmode:
	  if (TARGET_AVX)
	    gen = gen_avx_blendvps256;
	  break;
	case V4DFmode:
	  if (TARGET_AVX)
	    gen = gen_avx_blendvpd256;
	  break;
	case V32QImode:
	case V16HImode:
	case V8SImode:
	case V4DImode:
	  if (TARGET_AVX2)
	    {
	      gen = gen_avx2_pblendvb;
	      if (mode != V32QImode)
		d = gen_reg_rtx (V32QImode);
	      op_false = gen_lowpart (V32QImode, op_false);
	      op_true = gen_lowpart (V32QImode, op_true);
	      cmp = gen_lowpart (V32QImode, cmp);
	    }
	  break;

	case V64QImode:
	  gen = gen_avx512bw_blendmv64qi;
	  break;
	case V32HImode:
	  gen = gen_avx512bw_blendmv32hi;
	  break;
	case V16SImode:
	  gen = gen_avx512f_blendmv16si;
	  break;
	case V8DImode:
	  gen = gen_avx512f_blendmv8di;
	  break;
	case V8DFmode:
	  gen = gen_avx512f_blendmv8df;
	  break;
	case V16SFmode:
	  gen = gen_avx512f_blendmv16sf;
	  break;

	default:
	  break;
	}

      if (gen != NULL)
	{
	  emit_insn (gen (d, op_false, op_true, cmp));
	  if (d != dest)
	    emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
	}
      else
	{
	  op_true = force_reg (mode, op_true);

	  t2 = gen_reg_rtx (mode);
	  if (optimize)
	    t3 = gen_reg_rtx (mode);
	  else
	    t3 = dest;

	  x = gen_rtx_AND (mode, op_true, cmp);
	  emit_insn (gen_rtx_SET (t2, x));

	  x = gen_rtx_NOT (mode, cmp);
	  x = gen_rtx_AND (mode, x, op_false);
	  emit_insn (gen_rtx_SET (t3, x));

	  x = gen_rtx_IOR (mode, t3, t2);
	  emit_insn (gen_rtx_SET (dest, x));
	}
    }
}

/* Expand a floating-point conditional move.  Return true if successful.  */

bool
ix86_expand_fp_movcc (rtx operands[])
{
  machine_mode mode = GET_MODE (operands[0]);
  enum rtx_code code = GET_CODE (operands[1]);
  rtx tmp, compare_op;
  rtx op0 = XEXP (operands[1], 0);
  rtx op1 = XEXP (operands[1], 1);

  if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
    {
      machine_mode cmode;

      /* Since we've no cmove for sse registers, don't force bad register
	 allocation just to gain access to it.  Deny movcc when the
	 comparison mode doesn't match the move mode.  */
      cmode = GET_MODE (op0);
      if (cmode == VOIDmode)
	cmode = GET_MODE (op1);
      if (cmode != mode)
	return false;

      code = ix86_prepare_sse_fp_compare_args (operands[0], code, &op0, &op1);
      if (code == UNKNOWN)
	return false;

      if (ix86_expand_sse_fp_minmax (operands[0], code, op0, op1,
				     operands[2], operands[3]))
	return true;

      tmp = ix86_expand_sse_cmp (operands[0], code, op0, op1,
				 operands[2], operands[3]);
      ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
      return true;
    }

  if (GET_MODE (op0) == TImode
      || (GET_MODE (op0) == DImode
	  && !TARGET_64BIT))
    return false;

  /* The floating point conditional move instructions don't directly
     support conditions resulting from a signed integer comparison.  */

  compare_op = ix86_expand_compare (code, op0, op1);
  if (!fcmov_comparison_operator (compare_op, VOIDmode))
    {
      tmp = gen_reg_rtx (QImode);
      ix86_expand_setcc (tmp, code, op0, op1);

      compare_op = ix86_expand_compare (NE, tmp, const0_rtx);
    }

  emit_insn (gen_rtx_SET (operands[0],
			  gen_rtx_IF_THEN_ELSE (mode, compare_op,
						operands[2], operands[3])));

  return true;
}

/* Helper for ix86_cmp_code_to_pcmp_immediate for int modes.  */

static int
ix86_int_cmp_code_to_pcmp_immediate (enum rtx_code code)
{
  switch (code)
    {
    case EQ:
      return 0;
    case LT:
    case LTU:
      return 1;
    case LE:
    case LEU:
      return 2;
    case NE:
      return 4;
    case GE:
    case GEU:
      return 5;
    case GT:
    case GTU:
      return 6;
    default:
      gcc_unreachable ();
    }
}

/* Helper for ix86_cmp_code_to_pcmp_immediate for fp modes.  */

static int
ix86_fp_cmp_code_to_pcmp_immediate (enum rtx_code code)
{
  switch (code)
    {
    case EQ:
      return 0x00;
    case NE:
      return 0x04;
    case GT:
      return 0x0e;
    case LE:
      return 0x02;
    case GE:
      return 0x0d;
    case LT:
      return 0x01;
    case UNLE:
      return 0x0a;
    case UNLT:
      return 0x09;
    case UNGE:
      return 0x05;
    case UNGT:
      return 0x06;
    case UNEQ:
      return 0x18;
    case LTGT:
      return 0x0c;
    case ORDERED:
      return 0x07;
    case UNORDERED:
      return 0x03;
    default:
      gcc_unreachable ();
    }
}

/* Return immediate value to be used in UNSPEC_PCMP
   for comparison CODE in MODE.  */

static int
ix86_cmp_code_to_pcmp_immediate (enum rtx_code code, machine_mode mode)
{
  if (FLOAT_MODE_P (mode))
    return ix86_fp_cmp_code_to_pcmp_immediate (code);
  return ix86_int_cmp_code_to_pcmp_immediate (code);
}

/* Expand AVX-512 vector comparison.  */

bool
ix86_expand_mask_vec_cmp (rtx operands[])
{
  machine_mode mask_mode = GET_MODE (operands[0]);
  machine_mode cmp_mode = GET_MODE (operands[2]);
  enum rtx_code code = GET_CODE (operands[1]);
  rtx imm = GEN_INT (ix86_cmp_code_to_pcmp_immediate (code, cmp_mode));
  int unspec_code;
  rtx unspec;

  switch (code)
    {
    case LEU:
    case GTU:
    case GEU:
    case LTU:
      unspec_code = UNSPEC_UNSIGNED_PCMP;
      break;

    default:
      unspec_code = UNSPEC_PCMP;
    }

  unspec = gen_rtx_UNSPEC (mask_mode, gen_rtvec (3, operands[2],
						 operands[3], imm),
			   unspec_code);
  emit_insn (gen_rtx_SET (operands[0], unspec));

  return true;
}

/* Expand fp vector comparison.  */

bool
ix86_expand_fp_vec_cmp (rtx operands[])
{
  enum rtx_code code = GET_CODE (operands[1]);
  rtx cmp;

  code = ix86_prepare_sse_fp_compare_args (operands[0], code,
					   &operands[2], &operands[3]);
  if (code == UNKNOWN)
    {
      rtx temp;
      switch (GET_CODE (operands[1]))
	{
	case LTGT:
	  temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[2],
				      operands[3], NULL, NULL);
	  cmp = ix86_expand_sse_cmp (operands[0], NE, operands[2],
				     operands[3], NULL, NULL);
	  code = AND;
	  break;
	case UNEQ:
	  temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[2],
				      operands[3], NULL, NULL);
	  cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[2],
				     operands[3], NULL, NULL);
	  code = IOR;
	  break;
	default:
	  gcc_unreachable ();
	}
      cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
				 OPTAB_DIRECT);
    }
  else
    cmp = ix86_expand_sse_cmp (operands[0], code, operands[2], operands[3],
			       operands[1], operands[2]);

  if (operands[0] != cmp)
    emit_move_insn (operands[0], cmp);

  return true;
}

static rtx
ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
			 rtx op_true, rtx op_false, bool *negate)
{
  machine_mode data_mode = GET_MODE (dest);
  machine_mode mode = GET_MODE (cop0);
  rtx x;

  *negate = false;

  /* XOP supports all of the comparisons on all 128-bit vector int types.  */
  if (TARGET_XOP
      && (mode == V16QImode || mode == V8HImode
	  || mode == V4SImode || mode == V2DImode))
    ;
  else
    {
      /* Canonicalize the comparison to EQ, GT, GTU.  */
      switch (code)
	{
	case EQ:
	case GT:
	case GTU:
	  break;

	case NE:
	case LE:
	case LEU:
	  code = reverse_condition (code);
	  *negate = true;
	  break;

	case GE:
	case GEU:
	  code = reverse_condition (code);
	  *negate = true;
	  /* FALLTHRU */

	case LT:
	case LTU:
	  std::swap (cop0, cop1);
	  code = swap_condition (code);
	  break;

	default:
	  gcc_unreachable ();
	}

      /* Only SSE4.1/SSE4.2 supports V2DImode.  */
      if (mode == V2DImode)
	{
	  switch (code)
	    {
	    case EQ:
	      /* SSE4.1 supports EQ.  */
	      if (!TARGET_SSE4_1)
		return NULL;
	      break;

	    case GT:
	    case GTU:
	      /* SSE4.2 supports GT/GTU.  */
	      if (!TARGET_SSE4_2)
		return NULL;
	      break;

	    default:
	      gcc_unreachable ();
	    }
	}

      /* Unsigned parallel compare is not supported by the hardware.
	 Play some tricks to turn this into a signed comparison
	 against 0.  */
      if (code == GTU)
	{
	  cop0 = force_reg (mode, cop0);

	  switch (mode)
	    {
	    case V16SImode:
	    case V8DImode:
	    case V8SImode:
	    case V4DImode:
	    case V4SImode:
	    case V2DImode:
		{
		  rtx t1, t2, mask;
		  rtx (*gen_sub3) (rtx, rtx, rtx);

		  switch (mode)
		    {
		    case V16SImode: gen_sub3 = gen_subv16si3; break;
		    case V8DImode: gen_sub3 = gen_subv8di3; break;
		    case V8SImode: gen_sub3 = gen_subv8si3; break;
		    case V4DImode: gen_sub3 = gen_subv4di3; break;
		    case V4SImode: gen_sub3 = gen_subv4si3; break;
		    case V2DImode: gen_sub3 = gen_subv2di3; break;
		    default:
		      gcc_unreachable ();
		    }
		  /* Subtract (-(INT MAX) - 1) from both operands to make
		     them signed.  */
		  mask = ix86_build_signbit_mask (mode, true, false);
		  t1 = gen_reg_rtx (mode);
		  emit_insn (gen_sub3 (t1, cop0, mask));

		  t2 = gen_reg_rtx (mode);
		  emit_insn (gen_sub3 (t2, cop1, mask));

		  cop0 = t1;
		  cop1 = t2;
		  code = GT;
		}
	      break;

	    case V64QImode:
	    case V32HImode:
	    case V32QImode:
	    case V16HImode:
	    case V16QImode:
	    case V8HImode:
	      /* Perform a parallel unsigned saturating subtraction.  */
	      x = gen_reg_rtx (mode);
	      emit_insn (gen_rtx_SET (x, gen_rtx_US_MINUS (mode, cop0,
							   cop1)));

	      cop0 = x;
	      cop1 = CONST0_RTX (mode);
	      code = EQ;
	      *negate = !*negate;
	      break;

	    default:
	      gcc_unreachable ();
	    }
	}
    }

  if (*negate)
    std::swap (op_true, op_false);

  /* Allow the comparison to be done in one mode, but the movcc to
     happen in another mode.  */
  if (data_mode == mode)
    {
      x = ix86_expand_sse_cmp (dest, code, cop0, cop1,
			       op_true, op_false);
    }
  else
    {
      gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
      x = ix86_expand_sse_cmp (gen_reg_rtx (mode), code, cop0, cop1,
			       op_true, op_false);
      if (GET_MODE (x) == mode)
	x = gen_lowpart (data_mode, x);
    }

  return x;
}

/* Expand integer vector comparison.  */

bool
ix86_expand_int_vec_cmp (rtx operands[])
{
  rtx_code code = GET_CODE (operands[1]);
  bool negate = false;
  rtx cmp = ix86_expand_int_sse_cmp (operands[0], code, operands[2],
				     operands[3], NULL, NULL, &negate);

  if (!cmp)
    return false;

  if (negate)
    cmp = ix86_expand_int_sse_cmp (operands[0], EQ, cmp,
				   CONST0_RTX (GET_MODE (cmp)),
				   NULL, NULL, &negate);

  gcc_assert (!negate);

  if (operands[0] != cmp)
    emit_move_insn (operands[0], cmp);

  return true;
}

/* Expand a floating-point vector conditional move; a vcond operation
   rather than a movcc operation.  */

bool
ix86_expand_fp_vcond (rtx operands[])
{
  enum rtx_code code = GET_CODE (operands[3]);
  rtx cmp;

  code = ix86_prepare_sse_fp_compare_args (operands[0], code,
					   &operands[4], &operands[5]);
  if (code == UNKNOWN)
    {
      rtx temp;
      switch (GET_CODE (operands[3]))
	{
	case LTGT:
	  temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[4],
				      operands[5], operands[0], operands[0]);
	  cmp = ix86_expand_sse_cmp (operands[0], NE, operands[4],
				     operands[5], operands[1], operands[2]);
	  code = AND;
	  break;
	case UNEQ:
	  temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[4],
				      operands[5], operands[0], operands[0]);
	  cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[4],
				     operands[5], operands[1], operands[2]);
	  code = IOR;
	  break;
	default:
	  gcc_unreachable ();
	}
      cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
				 OPTAB_DIRECT);
      ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
      return true;
    }

  if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
				 operands[5], operands[1], operands[2]))
    return true;

  cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
			     operands[1], operands[2]);
  ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
  return true;
}

/* Expand a signed/unsigned integral vector conditional move.  */

bool
ix86_expand_int_vcond (rtx operands[])
{
  machine_mode data_mode = GET_MODE (operands[0]);
  machine_mode mode = GET_MODE (operands[4]);
  enum rtx_code code = GET_CODE (operands[3]);
  bool negate = false;
  rtx x, cop0, cop1;

  cop0 = operands[4];
  cop1 = operands[5];

  /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
     and x < 0 ? 1 : 0 into (unsigned) x >> 31.  */
  if ((code == LT || code == GE)
      && data_mode == mode
      && cop1 == CONST0_RTX (mode)
      && operands[1 + (code == LT)] == CONST0_RTX (data_mode)
      && GET_MODE_UNIT_SIZE (data_mode) > 1
      && GET_MODE_UNIT_SIZE (data_mode) <= 8
      && (GET_MODE_SIZE (data_mode) == 16
	  || (TARGET_AVX2 && GET_MODE_SIZE (data_mode) == 32)))
    {
      rtx negop = operands[2 - (code == LT)];
      int shift = GET_MODE_UNIT_BITSIZE (data_mode) - 1;
      if (negop == CONST1_RTX (data_mode))
	{
	  rtx res = expand_simple_binop (mode, LSHIFTRT, cop0, GEN_INT (shift),
					 operands[0], 1, OPTAB_DIRECT);
	  if (res != operands[0])
	    emit_move_insn (operands[0], res);
	  return true;
	}
      else if (GET_MODE_INNER (data_mode) != DImode
	       && vector_all_ones_operand (negop, data_mode))
	{
	  rtx res = expand_simple_binop (mode, ASHIFTRT, cop0, GEN_INT (shift),
					 operands[0], 0, OPTAB_DIRECT);
	  if (res != operands[0])
	    emit_move_insn (operands[0], res);
	  return true;
	}
    }

  if (!nonimmediate_operand (cop1, mode))
    cop1 = force_reg (mode, cop1);
  if (!general_operand (operands[1], data_mode))
    operands[1] = force_reg (data_mode, operands[1]);
  if (!general_operand (operands[2], data_mode))
    operands[2] = force_reg (data_mode, operands[2]);

  x = ix86_expand_int_sse_cmp (operands[0], code, cop0, cop1,
			       operands[1], operands[2], &negate);

  if (!x)
    return false;

  ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
			 operands[2-negate]);
  return true;
}

/* AVX512F does support 64-byte integer vector operations,
   thus the longest vector we are faced with is V64QImode.  */
#define MAX_VECT_LEN	64

struct expand_vec_perm_d
{
  rtx target, op0, op1;
  unsigned char perm[MAX_VECT_LEN];
  machine_mode vmode;
  unsigned char nelt;
  bool one_operand_p;
  bool testing_p;
};

static bool
ix86_expand_vec_perm_vpermi2 (rtx target, rtx op0, rtx mask, rtx op1,
			      struct expand_vec_perm_d *d)
{
  /* ix86_expand_vec_perm_vpermi2 is called from both const and non-const
     expander, so args are either in d, or in op0, op1 etc.  */
  machine_mode mode = GET_MODE (d ? d->op0 : op0);
  machine_mode maskmode = mode;
  rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;

  switch (mode)
    {
    case V8HImode:
      if (TARGET_AVX512VL && TARGET_AVX512BW)
	gen = gen_avx512vl_vpermi2varv8hi3;
      break;
    case V16HImode:
      if (TARGET_AVX512VL && TARGET_AVX512BW)
	gen = gen_avx512vl_vpermi2varv16hi3;
      break;
    case V64QImode:
      if (TARGET_AVX512VBMI)
	gen = gen_avx512bw_vpermi2varv64qi3;
      break;
    case V32HImode:
      if (TARGET_AVX512BW)
	gen = gen_avx512bw_vpermi2varv32hi3;
      break;
    case V4SImode:
      if (TARGET_AVX512VL)
	gen = gen_avx512vl_vpermi2varv4si3;
      break;
    case V8SImode:
      if (TARGET_AVX512VL)
	gen = gen_avx512vl_vpermi2varv8si3;
      break;
    case V16SImode:
      if (TARGET_AVX512F)
	gen = gen_avx512f_vpermi2varv16si3;
      break;
    case V4SFmode:
      if (TARGET_AVX512VL)
	{
	  gen = gen_avx512vl_vpermi2varv4sf3;
	  maskmode = V4SImode;
	}
      break;
    case V8SFmode:
      if (TARGET_AVX512VL)
	{
	  gen = gen_avx512vl_vpermi2varv8sf3;
	  maskmode = V8SImode;
	}
      break;
    case V16SFmode:
      if (TARGET_AVX512F)
	{
	  gen = gen_avx512f_vpermi2varv16sf3;
	  maskmode = V16SImode;
	}
      break;
    case V2DImode:
      if (TARGET_AVX512VL)
	gen = gen_avx512vl_vpermi2varv2di3;
      break;
    case V4DImode:
      if (TARGET_AVX512VL)
	gen = gen_avx512vl_vpermi2varv4di3;
      break;
    case V8DImode:
      if (TARGET_AVX512F)
	gen = gen_avx512f_vpermi2varv8di3;
      break;
    case V2DFmode:
      if (TARGET_AVX512VL)
	{
	  gen = gen_avx512vl_vpermi2varv2df3;
	  maskmode = V2DImode;
	}
      break;
    case V4DFmode:
      if (TARGET_AVX512VL)
	{
	  gen = gen_avx512vl_vpermi2varv4df3;
	  maskmode = V4DImode;
	}
      break;
    case V8DFmode:
      if (TARGET_AVX512F)
	{
	  gen = gen_avx512f_vpermi2varv8df3;
	  maskmode = V8DImode;
	}
      break;
    default:
      break;
    }

  if (gen == NULL)
    return false;

  /* ix86_expand_vec_perm_vpermi2 is called from both const and non-const
     expander, so args are either in d, or in op0, op1 etc.  */
  if (d)
    {
      rtx vec[64];
      target = d->target;
      op0 = d->op0;
      op1 = d->op1;
      for (int i = 0; i < d->nelt; ++i)
	vec[i] = GEN_INT (d->perm[i]);
      mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
    }

  emit_insn (gen (target, op0, force_reg (maskmode, mask), op1));
  return true;
}

/* Expand a variable vector permutation.  */

void
ix86_expand_vec_perm (rtx operands[])
{
  rtx target = operands[0];
  rtx op0 = operands[1];
  rtx op1 = operands[2];
  rtx mask = operands[3];
  rtx t1, t2, t3, t4, t5, t6, t7, t8, vt, vt2, vec[32];
  machine_mode mode = GET_MODE (op0);
  machine_mode maskmode = GET_MODE (mask);
  int w, e, i;
  bool one_operand_shuffle = rtx_equal_p (op0, op1);

  /* Number of elements in the vector.  */
  w = GET_MODE_NUNITS (mode);
  e = GET_MODE_UNIT_SIZE (mode);
  gcc_assert (w <= 64);

  if (TARGET_AVX512F && one_operand_shuffle)
    {
      rtx (*gen) (rtx, rtx, rtx) = NULL;
      switch (mode)
	{
	case V16SImode:
	  gen =gen_avx512f_permvarv16si;
	  break;
	case V16SFmode:
	  gen = gen_avx512f_permvarv16sf;
	  break;
	case V8DImode:
	  gen = gen_avx512f_permvarv8di;
	  break;
	case V8DFmode:
	  gen = gen_avx512f_permvarv8df;
	  break;
	default:
	  break;
	}
      if (gen != NULL)
	{
	  emit_insn (gen (target, op0, mask));
	  return;
	}
    }

  if (ix86_expand_vec_perm_vpermi2 (target, op0, mask, op1, NULL))
    return;

  if (TARGET_AVX2)
    {
      if (mode == V4DImode || mode == V4DFmode || mode == V16HImode)
	{
	  /* Unfortunately, the VPERMQ and VPERMPD instructions only support
	     an constant shuffle operand.  With a tiny bit of effort we can
	     use VPERMD instead.  A re-interpretation stall for V4DFmode is
	     unfortunate but there's no avoiding it.
	     Similarly for V16HImode we don't have instructions for variable
	     shuffling, while for V32QImode we can use after preparing suitable
	     masks vpshufb; vpshufb; vpermq; vpor.  */

	  if (mode == V16HImode)
	    {
	      maskmode = mode = V32QImode;
	      w = 32;
	      e = 1;
	    }
	  else
	    {
	      maskmode = mode = V8SImode;
	      w = 8;
	      e = 4;
	    }
	  t1 = gen_reg_rtx (maskmode);

	  /* Replicate the low bits of the V4DImode mask into V8SImode:
	       mask = { A B C D }
	       t1 = { A A B B C C D D }.  */
	  for (i = 0; i < w / 2; ++i)
	    vec[i*2 + 1] = vec[i*2] = GEN_INT (i * 2);
	  vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
	  vt = force_reg (maskmode, vt);
	  mask = gen_lowpart (maskmode, mask);
	  if (maskmode == V8SImode)
	    emit_insn (gen_avx2_permvarv8si (t1, mask, vt));
	  else
	    emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));

	  /* Multiply the shuffle indicies by two.  */
	  t1 = expand_simple_binop (maskmode, PLUS, t1, t1, t1, 1,
				    OPTAB_DIRECT);

	  /* Add one to the odd shuffle indicies:
		t1 = { A*2, A*2+1, B*2, B*2+1, ... }.  */
	  for (i = 0; i < w / 2; ++i)
	    {
	      vec[i * 2] = const0_rtx;
	      vec[i * 2 + 1] = const1_rtx;
	    }
	  vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
	  vt = validize_mem (force_const_mem (maskmode, vt));
	  t1 = expand_simple_binop (maskmode, PLUS, t1, vt, t1, 1,
				    OPTAB_DIRECT);

	  /* Continue as if V8SImode (resp. V32QImode) was used initially.  */
	  operands[3] = mask = t1;
	  target = gen_reg_rtx (mode);
	  op0 = gen_lowpart (mode, op0);
	  op1 = gen_lowpart (mode, op1);
	}

      switch (mode)
	{
	case V8SImode:
	  /* The VPERMD and VPERMPS instructions already properly ignore
	     the high bits of the shuffle elements.  No need for us to
	     perform an AND ourselves.  */
	  if (one_operand_shuffle)
	    {
	      emit_insn (gen_avx2_permvarv8si (target, op0, mask));
	      if (target != operands[0])
		emit_move_insn (operands[0],
				gen_lowpart (GET_MODE (operands[0]), target));
	    }
	  else
	    {
	      t1 = gen_reg_rtx (V8SImode);
	      t2 = gen_reg_rtx (V8SImode);
	      emit_insn (gen_avx2_permvarv8si (t1, op0, mask));
	      emit_insn (gen_avx2_permvarv8si (t2, op1, mask));
	      goto merge_two;
	    }
	  return;

	case V8SFmode:
	  mask = gen_lowpart (V8SImode, mask);
	  if (one_operand_shuffle)
	    emit_insn (gen_avx2_permvarv8sf (target, op0, mask));
	  else
	    {
	      t1 = gen_reg_rtx (V8SFmode);
	      t2 = gen_reg_rtx (V8SFmode);
	      emit_insn (gen_avx2_permvarv8sf (t1, op0, mask));
	      emit_insn (gen_avx2_permvarv8sf (t2, op1, mask));
	      goto merge_two;
	    }
	  return;

        case V4SImode:
	  /* By combining the two 128-bit input vectors into one 256-bit
	     input vector, we can use VPERMD and VPERMPS for the full
	     two-operand shuffle.  */
	  t1 = gen_reg_rtx (V8SImode);
	  t2 = gen_reg_rtx (V8SImode);
	  emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
	  emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
	  emit_insn (gen_avx2_permvarv8si (t1, t1, t2));
	  emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
	  return;

        case V4SFmode:
	  t1 = gen_reg_rtx (V8SFmode);
	  t2 = gen_reg_rtx (V8SImode);
	  mask = gen_lowpart (V4SImode, mask);
	  emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
	  emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
	  emit_insn (gen_avx2_permvarv8sf (t1, t1, t2));
	  emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
	  return;

	case V32QImode:
	  t1 = gen_reg_rtx (V32QImode);
	  t2 = gen_reg_rtx (V32QImode);
	  t3 = gen_reg_rtx (V32QImode);
	  vt2 = GEN_INT (-128);
	  for (i = 0; i < 32; i++)
	    vec[i] = vt2;
	  vt = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
	  vt = force_reg (V32QImode, vt);
	  for (i = 0; i < 32; i++)
	    vec[i] = i < 16 ? vt2 : const0_rtx;
	  vt2 = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
	  vt2 = force_reg (V32QImode, vt2);
	  /* From mask create two adjusted masks, which contain the same
	     bits as mask in the low 7 bits of each vector element.
	     The first mask will have the most significant bit clear
	     if it requests element from the same 128-bit lane
	     and MSB set if it requests element from the other 128-bit lane.
	     The second mask will have the opposite values of the MSB,
	     and additionally will have its 128-bit lanes swapped.
	     E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
	     t1   { 07 92 9e 09 ... | 17 19 85 1f ... } and
	     t3   { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
	     stands for other 12 bytes.  */
	  /* The bit whether element is from the same lane or the other
	     lane is bit 4, so shift it up by 3 to the MSB position.  */
	  t5 = gen_reg_rtx (V4DImode);
	  emit_insn (gen_ashlv4di3 (t5, gen_lowpart (V4DImode, mask),
				    GEN_INT (3)));
	  /* Clear MSB bits from the mask just in case it had them set.  */
	  emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
	  /* After this t1 will have MSB set for elements from other lane.  */
	  emit_insn (gen_xorv32qi3 (t1, gen_lowpart (V32QImode, t5), vt2));
	  /* Clear bits other than MSB.  */
	  emit_insn (gen_andv32qi3 (t1, t1, vt));
	  /* Or in the lower bits from mask into t3.  */
	  emit_insn (gen_iorv32qi3 (t3, t1, t2));
	  /* And invert MSB bits in t1, so MSB is set for elements from the same
	     lane.  */
	  emit_insn (gen_xorv32qi3 (t1, t1, vt));
	  /* Swap 128-bit lanes in t3.  */
	  t6 = gen_reg_rtx (V4DImode);
	  emit_insn (gen_avx2_permv4di_1 (t6, gen_lowpart (V4DImode, t3),
					  const2_rtx, GEN_INT (3),
					  const0_rtx, const1_rtx));
	  /* And or in the lower bits from mask into t1.  */
	  emit_insn (gen_iorv32qi3 (t1, t1, t2));
	  if (one_operand_shuffle)
	    {
	      /* Each of these shuffles will put 0s in places where
		 element from the other 128-bit lane is needed, otherwise
		 will shuffle in the requested value.  */
	      emit_insn (gen_avx2_pshufbv32qi3 (t3, op0,
						gen_lowpart (V32QImode, t6)));
	      emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
	      /* For t3 the 128-bit lanes are swapped again.  */
	      t7 = gen_reg_rtx (V4DImode);
	      emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t3),
					      const2_rtx, GEN_INT (3),
					      const0_rtx, const1_rtx));
	      /* And oring both together leads to the result.  */
	      emit_insn (gen_iorv32qi3 (target, t1,
					gen_lowpart (V32QImode, t7)));
	      if (target != operands[0])
		emit_move_insn (operands[0],
				gen_lowpart (GET_MODE (operands[0]), target));
	      return;
	    }

	  t4 = gen_reg_rtx (V32QImode);
	  /* Similarly to the above one_operand_shuffle code,
	     just for repeated twice for each operand.  merge_two:
	     code will merge the two results together.  */
	  emit_insn (gen_avx2_pshufbv32qi3 (t4, op0,
					    gen_lowpart (V32QImode, t6)));
	  emit_insn (gen_avx2_pshufbv32qi3 (t3, op1,
					    gen_lowpart (V32QImode, t6)));
	  emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
	  emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
	  t7 = gen_reg_rtx (V4DImode);
	  emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t4),
					  const2_rtx, GEN_INT (3),
					  const0_rtx, const1_rtx));
	  t8 = gen_reg_rtx (V4DImode);
	  emit_insn (gen_avx2_permv4di_1 (t8, gen_lowpart (V4DImode, t3),
					  const2_rtx, GEN_INT (3),
					  const0_rtx, const1_rtx));
	  emit_insn (gen_iorv32qi3 (t4, t2, gen_lowpart (V32QImode, t7)));
	  emit_insn (gen_iorv32qi3 (t3, t1, gen_lowpart (V32QImode, t8)));
	  t1 = t4;
	  t2 = t3;
	  goto merge_two;

	default:
	  gcc_assert (GET_MODE_SIZE (mode) <= 16);
	  break;
	}
    }

  if (TARGET_XOP)
    {
      /* The XOP VPPERM insn supports three inputs.  By ignoring the 
	 one_operand_shuffle special case, we avoid creating another
	 set of constant vectors in memory.  */
      one_operand_shuffle = false;

      /* mask = mask & {2*w-1, ...} */
      vt = GEN_INT (2*w - 1);
    }
  else
    {
      /* mask = mask & {w-1, ...} */
      vt = GEN_INT (w - 1);
    }

  for (i = 0; i < w; i++)
    vec[i] = vt;
  vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
  mask = expand_simple_binop (maskmode, AND, mask, vt,
			      NULL_RTX, 0, OPTAB_DIRECT);

  /* For non-QImode operations, convert the word permutation control
     into a byte permutation control.  */
  if (mode != V16QImode)
    {
      mask = expand_simple_binop (maskmode, ASHIFT, mask,
				  GEN_INT (exact_log2 (e)),
				  NULL_RTX, 0, OPTAB_DIRECT);

      /* Convert mask to vector of chars.  */
      mask = force_reg (V16QImode, gen_lowpart (V16QImode, mask));

      /* Replicate each of the input bytes into byte positions:
	 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
	 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
	 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}.  */
      for (i = 0; i < 16; ++i)
	vec[i] = GEN_INT (i/e * e);
      vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
      vt = validize_mem (force_const_mem (V16QImode, vt));
      if (TARGET_XOP)
	emit_insn (gen_xop_pperm (mask, mask, mask, vt));
      else
	emit_insn (gen_ssse3_pshufbv16qi3 (mask, mask, vt));

      /* Convert it into the byte positions by doing
	 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...}  */
      for (i = 0; i < 16; ++i)
	vec[i] = GEN_INT (i % e);
      vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
      vt = validize_mem (force_const_mem (V16QImode, vt));
      emit_insn (gen_addv16qi3 (mask, mask, vt));
    }

  /* The actual shuffle operations all operate on V16QImode.  */
  op0 = gen_lowpart (V16QImode, op0);
  op1 = gen_lowpart (V16QImode, op1);

  if (TARGET_XOP)
    {
      if (GET_MODE (target) != V16QImode)
	target = gen_reg_rtx (V16QImode);
      emit_insn (gen_xop_pperm (target, op0, op1, mask));
      if (target != operands[0])
	emit_move_insn (operands[0],
			gen_lowpart (GET_MODE (operands[0]), target));
    }
  else if (one_operand_shuffle)
    {
      if (GET_MODE (target) != V16QImode)
	target = gen_reg_rtx (V16QImode);
      emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
      if (target != operands[0])
	emit_move_insn (operands[0],
			gen_lowpart (GET_MODE (operands[0]), target));
    }
  else
    {
      rtx xops[6];
      bool ok;

      /* Shuffle the two input vectors independently.  */
      t1 = gen_reg_rtx (V16QImode);
      t2 = gen_reg_rtx (V16QImode);
      emit_insn (gen_ssse3_pshufbv16qi3 (t1, op0, mask));
      emit_insn (gen_ssse3_pshufbv16qi3 (t2, op1, mask));

 merge_two:
      /* Then merge them together.  The key is whether any given control
         element contained a bit set that indicates the second word.  */
      mask = operands[3];
      vt = GEN_INT (w);
      if (maskmode == V2DImode && !TARGET_SSE4_1)
	{
	  /* Without SSE4.1, we don't have V2DImode EQ.  Perform one
	     more shuffle to convert the V2DI input mask into a V4SI
	     input mask.  At which point the masking that expand_int_vcond
	     will work as desired.  */
	  rtx t3 = gen_reg_rtx (V4SImode);
	  emit_insn (gen_sse2_pshufd_1 (t3, gen_lowpart (V4SImode, mask),
				        const0_rtx, const0_rtx,
				        const2_rtx, const2_rtx));
	  mask = t3;
	  maskmode = V4SImode;
	  e = w = 4;
	}

      for (i = 0; i < w; i++)
	vec[i] = vt;
      vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
      vt = force_reg (maskmode, vt);
      mask = expand_simple_binop (maskmode, AND, mask, vt,
				  NULL_RTX, 0, OPTAB_DIRECT);

      if (GET_MODE (target) != mode)
	target = gen_reg_rtx (mode);
      xops[0] = target;
      xops[1] = gen_lowpart (mode, t2);
      xops[2] = gen_lowpart (mode, t1);
      xops[3] = gen_rtx_EQ (maskmode, mask, vt);
      xops[4] = mask;
      xops[5] = vt;
      ok = ix86_expand_int_vcond (xops);
      gcc_assert (ok);
      if (target != operands[0])
	emit_move_insn (operands[0],
			gen_lowpart (GET_MODE (operands[0]), target));
    }
}

/* Unpack OP[1] into the next wider integer vector type.  UNSIGNED_P is
   true if we should do zero extension, else sign extension.  HIGH_P is
   true if we want the N/2 high elements, else the low elements.  */

void
ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
{
  machine_mode imode = GET_MODE (src);
  rtx tmp;

  if (TARGET_SSE4_1)
    {
      rtx (*unpack)(rtx, rtx);
      rtx (*extract)(rtx, rtx) = NULL;
      machine_mode halfmode = BLKmode;

      switch (imode)
	{
	case V64QImode:
	  if (unsigned_p)
	    unpack = gen_avx512bw_zero_extendv32qiv32hi2;
	  else
	    unpack = gen_avx512bw_sign_extendv32qiv32hi2;
	  halfmode = V32QImode;
	  extract
	    = high_p ? gen_vec_extract_hi_v64qi : gen_vec_extract_lo_v64qi;
	  break;
	case V32QImode:
	  if (unsigned_p)
	    unpack = gen_avx2_zero_extendv16qiv16hi2;
	  else
	    unpack = gen_avx2_sign_extendv16qiv16hi2;
	  halfmode = V16QImode;
	  extract
	    = high_p ? gen_vec_extract_hi_v32qi : gen_vec_extract_lo_v32qi;
	  break;
	case V32HImode:
	  if (unsigned_p)
	    unpack = gen_avx512f_zero_extendv16hiv16si2;
	  else
	    unpack = gen_avx512f_sign_extendv16hiv16si2;
	  halfmode = V16HImode;
	  extract
	    = high_p ? gen_vec_extract_hi_v32hi : gen_vec_extract_lo_v32hi;
	  break;
	case V16HImode:
	  if (unsigned_p)
	    unpack = gen_avx2_zero_extendv8hiv8si2;
	  else
	    unpack = gen_avx2_sign_extendv8hiv8si2;
	  halfmode = V8HImode;
	  extract
	    = high_p ? gen_vec_extract_hi_v16hi : gen_vec_extract_lo_v16hi;
	  break;
	case V16SImode:
	  if (unsigned_p)
	    unpack = gen_avx512f_zero_extendv8siv8di2;
	  else
	    unpack = gen_avx512f_sign_extendv8siv8di2;
	  halfmode = V8SImode;
	  extract
	    = high_p ? gen_vec_extract_hi_v16si : gen_vec_extract_lo_v16si;
	  break;
	case V8SImode:
	  if (unsigned_p)
	    unpack = gen_avx2_zero_extendv4siv4di2;
	  else
	    unpack = gen_avx2_sign_extendv4siv4di2;
	  halfmode = V4SImode;
	  extract
	    = high_p ? gen_vec_extract_hi_v8si : gen_vec_extract_lo_v8si;
	  break;
	case V16QImode:
	  if (unsigned_p)
	    unpack = gen_sse4_1_zero_extendv8qiv8hi2;
	  else
	    unpack = gen_sse4_1_sign_extendv8qiv8hi2;
	  break;
	case V8HImode:
	  if (unsigned_p)
	    unpack = gen_sse4_1_zero_extendv4hiv4si2;
	  else
	    unpack = gen_sse4_1_sign_extendv4hiv4si2;
	  break;
	case V4SImode:
	  if (unsigned_p)
	    unpack = gen_sse4_1_zero_extendv2siv2di2;
	  else
	    unpack = gen_sse4_1_sign_extendv2siv2di2;
	  break;
	default:
	  gcc_unreachable ();
	}

      if (GET_MODE_SIZE (imode) >= 32)
	{
	  tmp = gen_reg_rtx (halfmode);
	  emit_insn (extract (tmp, src));
	}
      else if (high_p)
	{
	  /* Shift higher 8 bytes to lower 8 bytes.  */
	  tmp = gen_reg_rtx (V1TImode);
	  emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, src),
					 GEN_INT (64)));
	  tmp = gen_lowpart (imode, tmp);
	}
      else
	tmp = src;

      emit_insn (unpack (dest, tmp));
    }
  else
    {
      rtx (*unpack)(rtx, rtx, rtx);

      switch (imode)
	{
	case V16QImode:
	  if (high_p)
	    unpack = gen_vec_interleave_highv16qi;
	  else
	    unpack = gen_vec_interleave_lowv16qi;
	  break;
	case V8HImode:
	  if (high_p)
	    unpack = gen_vec_interleave_highv8hi;
	  else
	    unpack = gen_vec_interleave_lowv8hi;
	  break;
	case V4SImode:
	  if (high_p)
	    unpack = gen_vec_interleave_highv4si;
	  else
	    unpack = gen_vec_interleave_lowv4si;
	  break;
	default:
	  gcc_unreachable ();
	}

      if (unsigned_p)
	tmp = force_reg (imode, CONST0_RTX (imode));
      else
	tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
				   src, pc_rtx, pc_rtx);

      rtx tmp2 = gen_reg_rtx (imode);
      emit_insn (unpack (tmp2, src, tmp));
      emit_move_insn (dest, gen_lowpart (GET_MODE (dest), tmp2));
    }
}

/* Expand conditional increment or decrement using adb/sbb instructions.
   The default case using setcc followed by the conditional move can be
   done by generic code.  */
bool
ix86_expand_int_addcc (rtx operands[])
{
  enum rtx_code code = GET_CODE (operands[1]);
  rtx flags;
  rtx (*insn)(rtx, rtx, rtx, rtx, rtx);
  rtx compare_op;
  rtx val = const0_rtx;
  bool fpcmp = false;
  machine_mode mode;
  rtx op0 = XEXP (operands[1], 0);
  rtx op1 = XEXP (operands[1], 1);

  if (operands[3] != const1_rtx
      && operands[3] != constm1_rtx)
    return false;
  if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
     return false;
  code = GET_CODE (compare_op);

  flags = XEXP (compare_op, 0);

  if (GET_MODE (flags) == CCFPmode
      || GET_MODE (flags) == CCFPUmode)
    {
      fpcmp = true;
      code = ix86_fp_compare_code_to_integer (code);
    }

  if (code != LTU)
    {
      val = constm1_rtx;
      if (fpcmp)
	PUT_CODE (compare_op,
		  reverse_condition_maybe_unordered
		    (GET_CODE (compare_op)));
      else
	PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
    }

  mode = GET_MODE (operands[0]);

  /* Construct either adc or sbb insn.  */
  if ((code == LTU) == (operands[3] == constm1_rtx))
    {
      switch (mode)
	{
	  case QImode:
	    insn = gen_subqi3_carry;
	    break;
	  case HImode:
	    insn = gen_subhi3_carry;
	    break;
	  case SImode:
	    insn = gen_subsi3_carry;
	    break;
	  case DImode:
	    insn = gen_subdi3_carry;
	    break;
	  default:
	    gcc_unreachable ();
	}
    }
  else
    {
      switch (mode)
	{
	  case QImode:
	    insn = gen_addqi3_carry;
	    break;
	  case HImode:
	    insn = gen_addhi3_carry;
	    break;
	  case SImode:
	    insn = gen_addsi3_carry;
	    break;
	  case DImode:
	    insn = gen_adddi3_carry;
	    break;
	  default:
	    gcc_unreachable ();
	}
    }
  emit_insn (insn (operands[0], operands[2], val, flags, compare_op));

  return true;
}


/* Split operands 0 and 1 into half-mode parts.  Similar to split_double_mode,
   but works for floating pointer parameters and nonoffsetable memories.
   For pushes, it returns just stack offsets; the values will be saved
   in the right order.  Maximally three parts are generated.  */

static int
ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
{
  int size;

  if (!TARGET_64BIT)
    size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
  else
    size = (GET_MODE_SIZE (mode) + 4) / 8;

  gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
  gcc_assert (size >= 2 && size <= 4);

  /* Optimize constant pool reference to immediates.  This is used by fp
     moves, that force all constants to memory to allow combining.  */
  if (MEM_P (operand) && MEM_READONLY_P (operand))
    {
      rtx tmp = maybe_get_pool_constant (operand);
      if (tmp)
	operand = tmp;
    }

  if (MEM_P (operand) && !offsettable_memref_p (operand))
    {
      /* The only non-offsetable memories we handle are pushes.  */
      int ok = push_operand (operand, VOIDmode);

      gcc_assert (ok);

      operand = copy_rtx (operand);
      PUT_MODE (operand, word_mode);
      parts[0] = parts[1] = parts[2] = parts[3] = operand;
      return size;
    }

  if (GET_CODE (operand) == CONST_VECTOR)
    {
      machine_mode imode = int_mode_for_mode (mode);
      /* Caution: if we looked through a constant pool memory above,
	 the operand may actually have a different mode now.  That's
	 ok, since we want to pun this all the way back to an integer.  */
      operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
      gcc_assert (operand != NULL);
      mode = imode;
    }

  if (!TARGET_64BIT)
    {
      if (mode == DImode)
	split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
      else
	{
	  int i;

	  if (REG_P (operand))
	    {
	      gcc_assert (reload_completed);
	      for (i = 0; i < size; i++)
		parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
	    }
	  else if (offsettable_memref_p (operand))
	    {
	      operand = adjust_address (operand, SImode, 0);
	      parts[0] = operand;
	      for (i = 1; i < size; i++)
		parts[i] = adjust_address (operand, SImode, 4 * i);
	    }
	  else if (CONST_DOUBLE_P (operand))
	    {
	      const REAL_VALUE_TYPE *r;
	      long l[4];

	      r = CONST_DOUBLE_REAL_VALUE (operand);
	      switch (mode)
		{
		case TFmode:
		  real_to_target (l, r, mode);
		  parts[3] = gen_int_mode (l[3], SImode);
		  parts[2] = gen_int_mode (l[2], SImode);
		  break;
		case XFmode:
		  /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
		     long double may not be 80-bit.  */
		  real_to_target (l, r, mode);
		  parts[2] = gen_int_mode (l[2], SImode);
		  break;
		case DFmode:
		  REAL_VALUE_TO_TARGET_DOUBLE (*r, l);
		  break;
		default:
		  gcc_unreachable ();
		}
	      parts[1] = gen_int_mode (l[1], SImode);
	      parts[0] = gen_int_mode (l[0], SImode);
	    }
	  else
	    gcc_unreachable ();
	}
    }
  else
    {
      if (mode == TImode)
	split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
      if (mode == XFmode || mode == TFmode)
	{
	  machine_mode upper_mode = mode==XFmode ? SImode : DImode;
	  if (REG_P (operand))
	    {
	      gcc_assert (reload_completed);
	      parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
	      parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
	    }
	  else if (offsettable_memref_p (operand))
	    {
	      operand = adjust_address (operand, DImode, 0);
	      parts[0] = operand;
	      parts[1] = adjust_address (operand, upper_mode, 8);
	    }
	  else if (CONST_DOUBLE_P (operand))
	    {
	      long l[4];

	      real_to_target (l, CONST_DOUBLE_REAL_VALUE (operand), mode);

	      /* real_to_target puts 32-bit pieces in each long.  */
	      parts[0] = gen_int_mode ((l[0] & HOST_WIDE_INT_C (0xffffffff))
				       | ((l[1] & HOST_WIDE_INT_C (0xffffffff))
					  << 32), DImode);

	      if (upper_mode == SImode)
	        parts[1] = gen_int_mode (l[2], SImode);
	      else
	        parts[1]
		  = gen_int_mode ((l[2] & HOST_WIDE_INT_C (0xffffffff))
				  | ((l[3] & HOST_WIDE_INT_C (0xffffffff))
				     << 32), DImode);
	    }
	  else
	    gcc_unreachable ();
	}
    }

  return size;
}

/* Emit insns to perform a move or push of DI, DF, XF, and TF values.
   Return false when normal moves are needed; true when all required
   insns have been emitted.  Operands 2-4 contain the input values
   int the correct order; operands 5-7 contain the output values.  */

void
ix86_split_long_move (rtx operands[])
{
  rtx part[2][4];
  int nparts, i, j;
  int push = 0;
  int collisions = 0;
  machine_mode mode = GET_MODE (operands[0]);
  bool collisionparts[4];

  /* The DFmode expanders may ask us to move double.
     For 64bit target this is single move.  By hiding the fact
     here we simplify i386.md splitters.  */
  if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8)
    {
      /* Optimize constant pool reference to immediates.  This is used by
	 fp moves, that force all constants to memory to allow combining.  */

      if (MEM_P (operands[1])
	  && GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF
	  && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
	operands[1] = get_pool_constant (XEXP (operands[1], 0));
      if (push_operand (operands[0], VOIDmode))
	{
	  operands[0] = copy_rtx (operands[0]);
	  PUT_MODE (operands[0], word_mode);
	}
      else
        operands[0] = gen_lowpart (DImode, operands[0]);
      operands[1] = gen_lowpart (DImode, operands[1]);
      emit_move_insn (operands[0], operands[1]);
      return;
    }

  /* The only non-offsettable memory we handle is push.  */
  if (push_operand (operands[0], VOIDmode))
    push = 1;
  else
    gcc_assert (!MEM_P (operands[0])
		|| offsettable_memref_p (operands[0]));

  nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
  ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));

  /* When emitting push, take care for source operands on the stack.  */
  if (push && MEM_P (operands[1])
      && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
    {
      rtx src_base = XEXP (part[1][nparts - 1], 0);

      /* Compensate for the stack decrement by 4.  */
      if (!TARGET_64BIT && nparts == 3
	  && mode == XFmode && TARGET_128BIT_LONG_DOUBLE)
	src_base = plus_constant (Pmode, src_base, 4);

      /* src_base refers to the stack pointer and is
	 automatically decreased by emitted push.  */
      for (i = 0; i < nparts; i++)
	part[1][i] = change_address (part[1][i],
				     GET_MODE (part[1][i]), src_base);
    }

  /* We need to do copy in the right order in case an address register
     of the source overlaps the destination.  */
  if (REG_P (part[0][0]) && MEM_P (part[1][0]))
    {
      rtx tmp;

      for (i = 0; i < nparts; i++)
	{
	  collisionparts[i]
	    = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
	  if (collisionparts[i])
	    collisions++;
	}

      /* Collision in the middle part can be handled by reordering.  */
      if (collisions == 1 && nparts == 3 && collisionparts [1])
	{
	  std::swap (part[0][1], part[0][2]);
	  std::swap (part[1][1], part[1][2]);
	}
      else if (collisions == 1
	       && nparts == 4
	       && (collisionparts [1] || collisionparts [2]))
	{
	  if (collisionparts [1])
	    {
	      std::swap (part[0][1], part[0][2]);
	      std::swap (part[1][1], part[1][2]);
	    }
	  else
	    {
	      std::swap (part[0][2], part[0][3]);
	      std::swap (part[1][2], part[1][3]);
	    }
	}

      /* If there are more collisions, we can't handle it by reordering.
	 Do an lea to the last part and use only one colliding move.  */
      else if (collisions > 1)
	{
	  rtx base, addr, tls_base = NULL_RTX;

	  collisions = 1;

	  base = part[0][nparts - 1];

	  /* Handle the case when the last part isn't valid for lea.
	     Happens in 64-bit mode storing the 12-byte XFmode.  */
	  if (GET_MODE (base) != Pmode)
	    base = gen_rtx_REG (Pmode, REGNO (base));

	  addr = XEXP (part[1][0], 0);
	  if (TARGET_TLS_DIRECT_SEG_REFS)
	    {
	      struct ix86_address parts;
	      int ok = ix86_decompose_address (addr, &parts);
	      gcc_assert (ok);
	      if (parts.seg == DEFAULT_TLS_SEG_REG)
		{
		  /* It is not valid to use %gs: or %fs: in
		     lea though, so we need to remove it from the
		     address used for lea and add it to each individual
		     memory loads instead.  */
		  addr = copy_rtx (addr);
		  rtx *x = &addr;
		  while (GET_CODE (*x) == PLUS)
		    {
		      for (i = 0; i < 2; i++)
			{
			  rtx u = XEXP (*x, i);
			  if (GET_CODE (u) == ZERO_EXTEND)
			    u = XEXP (u, 0);
			  if (GET_CODE (u) == UNSPEC
			      && XINT (u, 1) == UNSPEC_TP)
			    {
			      tls_base = XEXP (*x, i);
			      *x = XEXP (*x, 1 - i);
			      break;
			    }
			}
		      if (tls_base)
			break;
		      x = &XEXP (*x, 0);
		    }
		  gcc_assert (tls_base);
		}
	    }
	  emit_insn (gen_rtx_SET (base, addr));
	  if (tls_base)
	    base = gen_rtx_PLUS (GET_MODE (base), base, tls_base);
	  part[1][0] = replace_equiv_address (part[1][0], base);
	  for (i = 1; i < nparts; i++)
	    {
	      if (tls_base)
		base = copy_rtx (base);
	      tmp = plus_constant (Pmode, base, UNITS_PER_WORD * i);
	      part[1][i] = replace_equiv_address (part[1][i], tmp);
	    }
	}
    }

  if (push)
    {
      if (!TARGET_64BIT)
	{
	  if (nparts == 3)
	    {
	      if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
                emit_insn (ix86_gen_add3 (stack_pointer_rtx,
					  stack_pointer_rtx, GEN_INT (-4)));
	      emit_move_insn (part[0][2], part[1][2]);
	    }
	  else if (nparts == 4)
	    {
	      emit_move_insn (part[0][3], part[1][3]);
	      emit_move_insn (part[0][2], part[1][2]);
	    }
	}
      else
	{
	  /* In 64bit mode we don't have 32bit push available.  In case this is
	     register, it is OK - we will just use larger counterpart.  We also
	     retype memory - these comes from attempt to avoid REX prefix on
	     moving of second half of TFmode value.  */
	  if (GET_MODE (part[1][1]) == SImode)
	    {
	      switch (GET_CODE (part[1][1]))
		{
		case MEM:
		  part[1][1] = adjust_address (part[1][1], DImode, 0);
		  break;

		case REG:
		  part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
		  break;

		default:
		  gcc_unreachable ();
		}

	      if (GET_MODE (part[1][0]) == SImode)
		part[1][0] = part[1][1];
	    }
	}
      emit_move_insn (part[0][1], part[1][1]);
      emit_move_insn (part[0][0], part[1][0]);
      return;
    }

  /* Choose correct order to not overwrite the source before it is copied.  */
  if ((REG_P (part[0][0])
       && REG_P (part[1][1])
       && (REGNO (part[0][0]) == REGNO (part[1][1])
	   || (nparts == 3
	       && REGNO (part[0][0]) == REGNO (part[1][2]))
	   || (nparts == 4
	       && REGNO (part[0][0]) == REGNO (part[1][3]))))
      || (collisions > 0
	  && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
    {
      for (i = 0, j = nparts - 1; i < nparts; i++, j--)
	{
	  operands[2 + i] = part[0][j];
	  operands[6 + i] = part[1][j];
	}
    }
  else
    {
      for (i = 0; i < nparts; i++)
	{
	  operands[2 + i] = part[0][i];
	  operands[6 + i] = part[1][i];
	}
    }

  /* If optimizing for size, attempt to locally unCSE nonzero constants.  */
  if (optimize_insn_for_size_p ())
    {
      for (j = 0; j < nparts - 1; j++)
	if (CONST_INT_P (operands[6 + j])
	    && operands[6 + j] != const0_rtx
	    && REG_P (operands[2 + j]))
	  for (i = j; i < nparts - 1; i++)
	    if (CONST_INT_P (operands[7 + i])
		&& INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
	      operands[7 + i] = operands[2 + j];
    }

  for (i = 0; i < nparts; i++)
    emit_move_insn (operands[2 + i], operands[6 + i]);

  return;
}

/* Helper function of ix86_split_ashl used to generate an SImode/DImode
   left shift by a constant, either using a single shift or
   a sequence of add instructions.  */

static void
ix86_expand_ashl_const (rtx operand, int count, machine_mode mode)
{
  rtx (*insn)(rtx, rtx, rtx);

  if (count == 1
      || (count * ix86_cost->add <= ix86_cost->shift_const
	  && !optimize_insn_for_size_p ()))
    {
      insn = mode == DImode ? gen_addsi3 : gen_adddi3;
      while (count-- > 0)
	emit_insn (insn (operand, operand, operand));
    }
  else
    {
      insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
      emit_insn (insn (operand, operand, GEN_INT (count)));
    }
}

void
ix86_split_ashl (rtx *operands, rtx scratch, machine_mode mode)
{
  rtx (*gen_ashl3)(rtx, rtx, rtx);
  rtx (*gen_shld)(rtx, rtx, rtx);
  int half_width = GET_MODE_BITSIZE (mode) >> 1;

  rtx low[2], high[2];
  int count;

  if (CONST_INT_P (operands[2]))
    {
      split_double_mode (mode, operands, 2, low, high);
      count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);

      if (count >= half_width)
	{
	  emit_move_insn (high[0], low[1]);
	  emit_move_insn (low[0], const0_rtx);

	  if (count > half_width)
	    ix86_expand_ashl_const (high[0], count - half_width, mode);
	}
      else
	{
	  gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;

	  if (!rtx_equal_p (operands[0], operands[1]))
	    emit_move_insn (operands[0], operands[1]);

	  emit_insn (gen_shld (high[0], low[0], GEN_INT (count)));
	  ix86_expand_ashl_const (low[0], count, mode);
	}
      return;
    }

  split_double_mode (mode, operands, 1, low, high);

  gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3;

  if (operands[1] == const1_rtx)
    {
      /* Assuming we've chosen a QImode capable registers, then 1 << N
	 can be done with two 32/64-bit shifts, no branches, no cmoves.  */
      if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
	{
	  rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);

	  ix86_expand_clear (low[0]);
	  ix86_expand_clear (high[0]);
	  emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width)));

	  d = gen_lowpart (QImode, low[0]);
	  d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
	  s = gen_rtx_EQ (QImode, flags, const0_rtx);
	  emit_insn (gen_rtx_SET (d, s));

	  d = gen_lowpart (QImode, high[0]);
	  d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
	  s = gen_rtx_NE (QImode, flags, const0_rtx);
	  emit_insn (gen_rtx_SET (d, s));
	}

      /* Otherwise, we can get the same results by manually performing
	 a bit extract operation on bit 5/6, and then performing the two
	 shifts.  The two methods of getting 0/1 into low/high are exactly
	 the same size.  Avoiding the shift in the bit extract case helps
	 pentium4 a bit; no one else seems to care much either way.  */
      else
	{
	  machine_mode half_mode;
	  rtx (*gen_lshr3)(rtx, rtx, rtx);
	  rtx (*gen_and3)(rtx, rtx, rtx);
	  rtx (*gen_xor3)(rtx, rtx, rtx);
	  HOST_WIDE_INT bits;
	  rtx x;

	  if (mode == DImode)
	    {
	      half_mode = SImode;
	      gen_lshr3 = gen_lshrsi3;
	      gen_and3 = gen_andsi3;
	      gen_xor3 = gen_xorsi3;
	      bits = 5;
	    }
	  else
	    {
	      half_mode = DImode;
	      gen_lshr3 = gen_lshrdi3;
	      gen_and3 = gen_anddi3;
	      gen_xor3 = gen_xordi3;
	      bits = 6;
	    }

	  if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
	    x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]);
	  else
	    x = gen_lowpart (half_mode, operands[2]);
	  emit_insn (gen_rtx_SET (high[0], x));

	  emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits)));
	  emit_insn (gen_and3 (high[0], high[0], const1_rtx));
	  emit_move_insn (low[0], high[0]);
	  emit_insn (gen_xor3 (low[0], low[0], const1_rtx));
	}

      emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
      emit_insn (gen_ashl3 (high[0], high[0], operands[2]));
      return;
    }

  if (operands[1] == constm1_rtx)
    {
      /* For -1 << N, we can avoid the shld instruction, because we
	 know that we're shifting 0...31/63 ones into a -1.  */
      emit_move_insn (low[0], constm1_rtx);
      if (optimize_insn_for_size_p ())
	emit_move_insn (high[0], low[0]);
      else
	emit_move_insn (high[0], constm1_rtx);
    }
  else
    {
      gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;

      if (!rtx_equal_p (operands[0], operands[1]))
	emit_move_insn (operands[0], operands[1]);

      split_double_mode (mode, operands, 1, low, high);
      emit_insn (gen_shld (high[0], low[0], operands[2]));
    }

  emit_insn (gen_ashl3 (low[0], low[0], operands[2]));

  if (TARGET_CMOVE && scratch)
    {
      rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
	= mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;

      ix86_expand_clear (scratch);
      emit_insn (gen_x86_shift_adj_1 (high[0], low[0], operands[2], scratch));
    }
  else
    {
      rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx)
	= mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2;

      emit_insn (gen_x86_shift_adj_2 (high[0], low[0], operands[2]));
    }
}

void
ix86_split_ashr (rtx *operands, rtx scratch, machine_mode mode)
{
  rtx (*gen_ashr3)(rtx, rtx, rtx)
    = mode == DImode ? gen_ashrsi3 : gen_ashrdi3;
  rtx (*gen_shrd)(rtx, rtx, rtx);
  int half_width = GET_MODE_BITSIZE (mode) >> 1;

  rtx low[2], high[2];
  int count;

  if (CONST_INT_P (operands[2]))
    {
      split_double_mode (mode, operands, 2, low, high);
      count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);

      if (count == GET_MODE_BITSIZE (mode) - 1)
	{
	  emit_move_insn (high[0], high[1]);
	  emit_insn (gen_ashr3 (high[0], high[0],
				GEN_INT (half_width - 1)));
	  emit_move_insn (low[0], high[0]);

	}
      else if (count >= half_width)
	{
	  emit_move_insn (low[0], high[1]);
	  emit_move_insn (high[0], low[0]);
	  emit_insn (gen_ashr3 (high[0], high[0],
				GEN_INT (half_width - 1)));

	  if (count > half_width)
	    emit_insn (gen_ashr3 (low[0], low[0],
				  GEN_INT (count - half_width)));
	}
      else
	{
	  gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;

	  if (!rtx_equal_p (operands[0], operands[1]))
	    emit_move_insn (operands[0], operands[1]);

	  emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
	  emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count)));
	}
    }
  else
    {
      gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;

     if (!rtx_equal_p (operands[0], operands[1]))
	emit_move_insn (operands[0], operands[1]);

      split_double_mode (mode, operands, 1, low, high);

      emit_insn (gen_shrd (low[0], high[0], operands[2]));
      emit_insn (gen_ashr3 (high[0], high[0], operands[2]));

      if (TARGET_CMOVE && scratch)
	{
	  rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
	    = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;

	  emit_move_insn (scratch, high[0]);
	  emit_insn (gen_ashr3 (scratch, scratch,
				GEN_INT (half_width - 1)));
	  emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
					  scratch));
	}
      else
	{
	  rtx (*gen_x86_shift_adj_3)(rtx, rtx, rtx)
	    = mode == DImode ? gen_x86_shiftsi_adj_3 : gen_x86_shiftdi_adj_3;

	  emit_insn (gen_x86_shift_adj_3 (low[0], high[0], operands[2]));
	}
    }
}

void
ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
{
  rtx (*gen_lshr3)(rtx, rtx, rtx)
    = mode == DImode ? gen_lshrsi3 : gen_lshrdi3;
  rtx (*gen_shrd)(rtx, rtx, rtx);
  int half_width = GET_MODE_BITSIZE (mode) >> 1;

  rtx low[2], high[2];
  int count;

  if (CONST_INT_P (operands[2]))
    {
      split_double_mode (mode, operands, 2, low, high);
      count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);

      if (count >= half_width)
	{
	  emit_move_insn (low[0], high[1]);
	  ix86_expand_clear (high[0]);

	  if (count > half_width)
	    emit_insn (gen_lshr3 (low[0], low[0],
				  GEN_INT (count - half_width)));
	}
      else
	{
	  gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;

	  if (!rtx_equal_p (operands[0], operands[1]))
	    emit_move_insn (operands[0], operands[1]);

	  emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
	  emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count)));
	}
    }
  else
    {
      gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;

      if (!rtx_equal_p (operands[0], operands[1]))
	emit_move_insn (operands[0], operands[1]);

      split_double_mode (mode, operands, 1, low, high);

      emit_insn (gen_shrd (low[0], high[0], operands[2]));
      emit_insn (gen_lshr3 (high[0], high[0], operands[2]));

      if (TARGET_CMOVE && scratch)
	{
	  rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
	    = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;

	  ix86_expand_clear (scratch);
	  emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
					  scratch));
	}
      else
	{
	  rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx)
	    = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2;

	  emit_insn (gen_x86_shift_adj_2 (low[0], high[0], operands[2]));
	}
    }
}

/* Predict just emitted jump instruction to be taken with probability PROB.  */
static void
predict_jump (int prob)
{
  rtx_insn *insn = get_last_insn ();
  gcc_assert (JUMP_P (insn));
  add_int_reg_note (insn, REG_BR_PROB, prob);
}

/* Helper function for the string operations below.  Dest VARIABLE whether
   it is aligned to VALUE bytes.  If true, jump to the label.  */
static rtx_code_label *
ix86_expand_aligntest (rtx variable, int value, bool epilogue)
{
  rtx_code_label *label = gen_label_rtx ();
  rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
  if (GET_MODE (variable) == DImode)
    emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
  else
    emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
  emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
			   1, label);
  if (epilogue)
    predict_jump (REG_BR_PROB_BASE * 50 / 100);
  else
    predict_jump (REG_BR_PROB_BASE * 90 / 100);
  return label;
}

/* Adjust COUNTER by the VALUE.  */
static void
ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
{
  rtx (*gen_add)(rtx, rtx, rtx)
    = GET_MODE (countreg) == DImode ? gen_adddi3 : gen_addsi3;

  emit_insn (gen_add (countreg, countreg, GEN_INT (-value)));
}

/* Zero extend possibly SImode EXP to Pmode register.  */
rtx
ix86_zero_extend_to_Pmode (rtx exp)
{
  return force_reg (Pmode, convert_to_mode (Pmode, exp, 1));
}

/* Divide COUNTREG by SCALE.  */
static rtx
scale_counter (rtx countreg, int scale)
{
  rtx sc;

  if (scale == 1)
    return countreg;
  if (CONST_INT_P (countreg))
    return GEN_INT (INTVAL (countreg) / scale);
  gcc_assert (REG_P (countreg));

  sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
			    GEN_INT (exact_log2 (scale)),
			    NULL, 1, OPTAB_DIRECT);
  return sc;
}

/* Return mode for the memcpy/memset loop counter.  Prefer SImode over
   DImode for constant loop counts.  */

static machine_mode
counter_mode (rtx count_exp)
{
  if (GET_MODE (count_exp) != VOIDmode)
    return GET_MODE (count_exp);
  if (!CONST_INT_P (count_exp))
    return Pmode;
  if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
    return DImode;
  return SImode;
}

/* Copy the address to a Pmode register.  This is used for x32 to
   truncate DImode TLS address to a SImode register. */

static rtx
ix86_copy_addr_to_reg (rtx addr)
{
  rtx reg;
  if (GET_MODE (addr) == Pmode || GET_MODE (addr) == VOIDmode)
    {
      reg = copy_addr_to_reg (addr);
      REG_POINTER (reg) = 1;
      return reg;
    }
  else
    {
      gcc_assert (GET_MODE (addr) == DImode && Pmode == SImode);
      reg = copy_to_mode_reg (DImode, addr);
      REG_POINTER (reg) = 1;
      return gen_rtx_SUBREG (SImode, reg, 0);
    }
}

/* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
   to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
   specified in bytes.  When ISSETMEM is TRUE, output the equivalent loop to set
   memory by VALUE (supposed to be in MODE).

   The size is rounded down to whole number of chunk size moved at once.
   SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info.  */


static void
expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
			       rtx destptr, rtx srcptr, rtx value,
			       rtx count, machine_mode mode, int unroll,
			       int expected_size, bool issetmem)
{
  rtx_code_label *out_label, *top_label;
  rtx iter, tmp;
  machine_mode iter_mode = counter_mode (count);
  int piece_size_n = GET_MODE_SIZE (mode) * unroll;
  rtx piece_size = GEN_INT (piece_size_n);
  rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
  rtx size;
  int i;

  top_label = gen_label_rtx ();
  out_label = gen_label_rtx ();
  iter = gen_reg_rtx (iter_mode);

  size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
			      NULL, 1, OPTAB_DIRECT);
  /* Those two should combine.  */
  if (piece_size == const1_rtx)
    {
      emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
			       true, out_label);
      predict_jump (REG_BR_PROB_BASE * 10 / 100);
    }
  emit_move_insn (iter, const0_rtx);

  emit_label (top_label);

  tmp = convert_modes (Pmode, iter_mode, iter, true);

  /* This assert could be relaxed - in this case we'll need to compute
     smallest power of two, containing in PIECE_SIZE_N and pass it to
     offset_address.  */
  gcc_assert ((piece_size_n & (piece_size_n - 1)) == 0);
  destmem = offset_address (destmem, tmp, piece_size_n);
  destmem = adjust_address (destmem, mode, 0);

  if (!issetmem)
    {
      srcmem = offset_address (srcmem, copy_rtx (tmp), piece_size_n);
      srcmem = adjust_address (srcmem, mode, 0);

      /* When unrolling for chips that reorder memory reads and writes,
	 we can save registers by using single temporary.
	 Also using 4 temporaries is overkill in 32bit mode.  */
      if (!TARGET_64BIT && 0)
	{
	  for (i = 0; i < unroll; i++)
	    {
	      if (i)
		{
		  destmem =
		    adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
		  srcmem =
		    adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
		}
	      emit_move_insn (destmem, srcmem);
	    }
	}
      else
	{
	  rtx tmpreg[4];
	  gcc_assert (unroll <= 4);
	  for (i = 0; i < unroll; i++)
	    {
	      tmpreg[i] = gen_reg_rtx (mode);
	      if (i)
		{
		  srcmem =
		    adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
		}
	      emit_move_insn (tmpreg[i], srcmem);
	    }
	  for (i = 0; i < unroll; i++)
	    {
	      if (i)
		{
		  destmem =
		    adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
		}
	      emit_move_insn (destmem, tmpreg[i]);
	    }
	}
    }
  else
    for (i = 0; i < unroll; i++)
      {
	if (i)
	  destmem =
	    adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
	emit_move_insn (destmem, value);
      }

  tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
			     true, OPTAB_LIB_WIDEN);
  if (tmp != iter)
    emit_move_insn (iter, tmp);

  emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
			   true, top_label);
  if (expected_size != -1)
    {
      expected_size /= GET_MODE_SIZE (mode) * unroll;
      if (expected_size == 0)
	predict_jump (0);
      else if (expected_size > REG_BR_PROB_BASE)
	predict_jump (REG_BR_PROB_BASE - 1);
      else
        predict_jump (REG_BR_PROB_BASE - (REG_BR_PROB_BASE + expected_size / 2) / expected_size);
    }
  else
    predict_jump (REG_BR_PROB_BASE * 80 / 100);
  iter = ix86_zero_extend_to_Pmode (iter);
  tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
			     true, OPTAB_LIB_WIDEN);
  if (tmp != destptr)
    emit_move_insn (destptr, tmp);
  if (!issetmem)
    {
      tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
				 true, OPTAB_LIB_WIDEN);
      if (tmp != srcptr)
	emit_move_insn (srcptr, tmp);
    }
  emit_label (out_label);
}

/* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
   When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
   When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
   For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
   ORIG_VALUE is the original value passed to memset to fill the memory with.
   Other arguments have same meaning as for previous function.  */

static void
expand_set_or_movmem_via_rep (rtx destmem, rtx srcmem,
			   rtx destptr, rtx srcptr, rtx value, rtx orig_value,
			   rtx count,
			   machine_mode mode, bool issetmem)
{
  rtx destexp;
  rtx srcexp;
  rtx countreg;
  HOST_WIDE_INT rounded_count;

  /* If possible, it is shorter to use rep movs.
     TODO: Maybe it is better to move this logic to decide_alg.  */
  if (mode == QImode && CONST_INT_P (count) && !(INTVAL (count) & 3)
      && (!issetmem || orig_value == const0_rtx))
    mode = SImode;

  if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
    destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);

  countreg = ix86_zero_extend_to_Pmode (scale_counter (count,
						       GET_MODE_SIZE (mode)));
  if (mode != QImode)
    {
      destexp = gen_rtx_ASHIFT (Pmode, countreg,
				GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
      destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
    }
  else
    destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
  if ((!issetmem || orig_value == const0_rtx) && CONST_INT_P (count))
    {
      rounded_count
	= ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
      destmem = shallow_copy_rtx (destmem);
      set_mem_size (destmem, rounded_count);
    }
  else if (MEM_SIZE_KNOWN_P (destmem))
    clear_mem_size (destmem);

  if (issetmem)
    {
      value = force_reg (mode, gen_lowpart (mode, value));
      emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
    }
  else
    {
      if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
	srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
      if (mode != QImode)
	{
	  srcexp = gen_rtx_ASHIFT (Pmode, countreg,
				   GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
	  srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
	}
      else
	srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
      if (CONST_INT_P (count))
	{
	  rounded_count
	    = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
	  srcmem = shallow_copy_rtx (srcmem);
	  set_mem_size (srcmem, rounded_count);
	}
      else
	{
	  if (MEM_SIZE_KNOWN_P (srcmem))
	    clear_mem_size (srcmem);
	}
      emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
			      destexp, srcexp));
    }
}

/* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
   DESTMEM.
   SRC is passed by pointer to be updated on return.
   Return value is updated DST.  */
static rtx
emit_memmov (rtx destmem, rtx *srcmem, rtx destptr, rtx srcptr,
	     HOST_WIDE_INT size_to_move)
{
  rtx dst = destmem, src = *srcmem, adjust, tempreg;
  enum insn_code code;
  machine_mode move_mode;
  int piece_size, i;

  /* Find the widest mode in which we could perform moves.
     Start with the biggest power of 2 less than SIZE_TO_MOVE and half
     it until move of such size is supported.  */
  piece_size = 1 << floor_log2 (size_to_move);
  move_mode = mode_for_size (piece_size * BITS_PER_UNIT, MODE_INT, 0);
  code = optab_handler (mov_optab, move_mode);
  while (code == CODE_FOR_nothing && piece_size > 1)
    {
      piece_size >>= 1;
      move_mode = mode_for_size (piece_size * BITS_PER_UNIT, MODE_INT, 0);
      code = optab_handler (mov_optab, move_mode);
    }

  /* Find the corresponding vector mode with the same size as MOVE_MODE.
     MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.).  */
  if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
    {
      int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
      move_mode = mode_for_vector (word_mode, nunits);
      code = optab_handler (mov_optab, move_mode);
      if (code == CODE_FOR_nothing)
	{
	  move_mode = word_mode;
	  piece_size = GET_MODE_SIZE (move_mode);
	  code = optab_handler (mov_optab, move_mode);
	}
    }
  gcc_assert (code != CODE_FOR_nothing);

  dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
  src = adjust_automodify_address_nv (src, move_mode, srcptr, 0);

  /* Emit moves.  We'll need SIZE_TO_MOVE/PIECE_SIZES moves.  */
  gcc_assert (size_to_move % piece_size == 0);
  adjust = GEN_INT (piece_size);
  for (i = 0; i < size_to_move; i += piece_size)
    {
      /* We move from memory to memory, so we'll need to do it via
	 a temporary register.  */
      tempreg = gen_reg_rtx (move_mode);
      emit_insn (GEN_FCN (code) (tempreg, src));
      emit_insn (GEN_FCN (code) (dst, tempreg));

      emit_move_insn (destptr,
		      gen_rtx_PLUS (Pmode, copy_rtx (destptr), adjust));
      emit_move_insn (srcptr,
		      gen_rtx_PLUS (Pmode, copy_rtx (srcptr), adjust));

      dst = adjust_automodify_address_nv (dst, move_mode, destptr,
					  piece_size);
      src = adjust_automodify_address_nv (src, move_mode, srcptr,
					  piece_size);
    }

  /* Update DST and SRC rtx.  */
  *srcmem = src;
  return dst;
}

/* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST.  */
static void
expand_movmem_epilogue (rtx destmem, rtx srcmem,
			rtx destptr, rtx srcptr, rtx count, int max_size)
{
  rtx src, dest;
  if (CONST_INT_P (count))
    {
      HOST_WIDE_INT countval = INTVAL (count);
      HOST_WIDE_INT epilogue_size = countval % max_size;
      int i;

      /* For now MAX_SIZE should be a power of 2.  This assert could be
	 relaxed, but it'll require a bit more complicated epilogue
	 expanding.  */
      gcc_assert ((max_size & (max_size - 1)) == 0);
      for (i = max_size; i >= 1; i >>= 1)
	{
	  if (epilogue_size & i)
	    destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
	}
      return;
    }
  if (max_size > 8)
    {
      count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
				    count, 1, OPTAB_DIRECT);
      expand_set_or_movmem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
				     count, QImode, 1, 4, false);
      return;
    }

  /* When there are stringops, we can cheaply increase dest and src pointers.
     Otherwise we save code size by maintaining offset (zero is readily
     available from preceding rep operation) and using x86 addressing modes.
   */
  if (TARGET_SINGLE_STRINGOP)
    {
      if (max_size > 4)
	{
	  rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
	  src = change_address (srcmem, SImode, srcptr);
	  dest = change_address (destmem, SImode, destptr);
	  emit_insn (gen_strmov (destptr, dest, srcptr, src));
	  emit_label (label);
	  LABEL_NUSES (label) = 1;
	}
      if (max_size > 2)
	{
	  rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
	  src = change_address (srcmem, HImode, srcptr);
	  dest = change_address (destmem, HImode, destptr);
	  emit_insn (gen_strmov (destptr, dest, srcptr, src));
	  emit_label (label);
	  LABEL_NUSES (label) = 1;
	}
      if (max_size > 1)
	{
	  rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
	  src = change_address (srcmem, QImode, srcptr);
	  dest = change_address (destmem, QImode, destptr);
	  emit_insn (gen_strmov (destptr, dest, srcptr, src));
	  emit_label (label);
	  LABEL_NUSES (label) = 1;
	}
    }
  else
    {
      rtx offset = force_reg (Pmode, const0_rtx);
      rtx tmp;

      if (max_size > 4)
	{
	  rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
	  src = change_address (srcmem, SImode, srcptr);
	  dest = change_address (destmem, SImode, destptr);
	  emit_move_insn (dest, src);
	  tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
				     true, OPTAB_LIB_WIDEN);
	  if (tmp != offset)
	    emit_move_insn (offset, tmp);
	  emit_label (label);
	  LABEL_NUSES (label) = 1;
	}
      if (max_size > 2)
	{
	  rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
	  tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
	  src = change_address (srcmem, HImode, tmp);
	  tmp = gen_rtx_PLUS (Pmode, destptr, offset);
	  dest = change_address (destmem, HImode, tmp);
	  emit_move_insn (dest, src);
	  tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
				     true, OPTAB_LIB_WIDEN);
	  if (tmp != offset)
	    emit_move_insn (offset, tmp);
	  emit_label (label);
	  LABEL_NUSES (label) = 1;
	}
      if (max_size > 1)
	{
	  rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
	  tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
	  src = change_address (srcmem, QImode, tmp);
	  tmp = gen_rtx_PLUS (Pmode, destptr, offset);
	  dest = change_address (destmem, QImode, tmp);
	  emit_move_insn (dest, src);
	  emit_label (label);
	  LABEL_NUSES (label) = 1;
	}
    }
}

/* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
   with value PROMOTED_VAL.
   SRC is passed by pointer to be updated on return.
   Return value is updated DST.  */
static rtx
emit_memset (rtx destmem, rtx destptr, rtx promoted_val,
	     HOST_WIDE_INT size_to_move)
{
  rtx dst = destmem, adjust;
  enum insn_code code;
  machine_mode move_mode;
  int piece_size, i;

  /* Find the widest mode in which we could perform moves.
     Start with the biggest power of 2 less than SIZE_TO_MOVE and half
     it until move of such size is supported.  */
  move_mode = GET_MODE (promoted_val);
  if (move_mode == VOIDmode)
    move_mode = QImode;
  if (size_to_move < GET_MODE_SIZE (move_mode))
    {
      move_mode = mode_for_size (size_to_move * BITS_PER_UNIT, MODE_INT, 0);
      promoted_val = gen_lowpart (move_mode, promoted_val);
    }
  piece_size = GET_MODE_SIZE (move_mode);
  code = optab_handler (mov_optab, move_mode);
  gcc_assert (code != CODE_FOR_nothing && promoted_val != NULL_RTX);

  dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);

  /* Emit moves.  We'll need SIZE_TO_MOVE/PIECE_SIZES moves.  */
  gcc_assert (size_to_move % piece_size == 0);
  adjust = GEN_INT (piece_size);
  for (i = 0; i < size_to_move; i += piece_size)
    {
      if (piece_size <= GET_MODE_SIZE (word_mode))
	{
	  emit_insn (gen_strset (destptr, dst, promoted_val));
	  dst = adjust_automodify_address_nv (dst, move_mode, destptr,
					      piece_size);
	  continue;
	}

      emit_insn (GEN_FCN (code) (dst, promoted_val));

      emit_move_insn (destptr,
		      gen_rtx_PLUS (Pmode, copy_rtx (destptr), adjust));

      dst = adjust_automodify_address_nv (dst, move_mode, destptr,
					  piece_size);
    }

  /* Update DST rtx.  */
  return dst;
}
/* Output code to set at most count & (max_size - 1) bytes starting by DEST.  */
static void
expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
				 rtx count, int max_size)
{
  count =
    expand_simple_binop (counter_mode (count), AND, count,
			 GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
  expand_set_or_movmem_via_loop (destmem, NULL, destptr, NULL,
				 gen_lowpart (QImode, value), count, QImode,
				 1, max_size / 2, true);
}

/* Output code to set at most count & (max_size - 1) bytes starting by DEST.  */
static void
expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
			rtx count, int max_size)
{
  rtx dest;

  if (CONST_INT_P (count))
    {
      HOST_WIDE_INT countval = INTVAL (count);
      HOST_WIDE_INT epilogue_size = countval % max_size;
      int i;

      /* For now MAX_SIZE should be a power of 2.  This assert could be
	 relaxed, but it'll require a bit more complicated epilogue
	 expanding.  */
      gcc_assert ((max_size & (max_size - 1)) == 0);
      for (i = max_size; i >= 1; i >>= 1)
	{
	  if (epilogue_size & i)
	    {
	      if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
		destmem = emit_memset (destmem, destptr, vec_value, i);
	      else
		destmem = emit_memset (destmem, destptr, value, i);
	    }
	}
      return;
    }
  if (max_size > 32)
    {
      expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
      return;
    }
  if (max_size > 16)
    {
      rtx_code_label *label = ix86_expand_aligntest (count, 16, true);
      if (TARGET_64BIT)
	{
	  dest = change_address (destmem, DImode, destptr);
	  emit_insn (gen_strset (destptr, dest, value));
	  dest = adjust_automodify_address_nv (dest, DImode, destptr, 8);
	  emit_insn (gen_strset (destptr, dest, value));
	}
      else
	{
	  dest = change_address (destmem, SImode, destptr);
	  emit_insn (gen_strset (destptr, dest, value));
	  dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
	  emit_insn (gen_strset (destptr, dest, value));
	  dest = adjust_automodify_address_nv (dest, SImode, destptr, 8);
	  emit_insn (gen_strset (destptr, dest, value));
	  dest = adjust_automodify_address_nv (dest, SImode, destptr, 12);
	  emit_insn (gen_strset (destptr, dest, value));
	}
      emit_label (label);
      LABEL_NUSES (label) = 1;
    }
  if (max_size > 8)
    {
      rtx_code_label *label = ix86_expand_aligntest (count, 8, true);
      if (TARGET_64BIT)
	{
	  dest = change_address (destmem, DImode, destptr);
	  emit_insn (gen_strset (destptr, dest, value));
	}
      else
	{
	  dest = change_address (destmem, SImode, destptr);
	  emit_insn (gen_strset (destptr, dest, value));
	  dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
	  emit_insn (gen_strset (destptr, dest, value));
	}
      emit_label (label);
      LABEL_NUSES (label) = 1;
    }
  if (max_size > 4)
    {
      rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
      dest = change_address (destmem, SImode, destptr);
      emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
      emit_label (label);
      LABEL_NUSES (label) = 1;
    }
  if (max_size > 2)
    {
      rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
      dest = change_address (destmem, HImode, destptr);
      emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
      emit_label (label);
      LABEL_NUSES (label) = 1;
    }
  if (max_size > 1)
    {
      rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
      dest = change_address (destmem, QImode, destptr);
      emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
      emit_label (label);
      LABEL_NUSES (label) = 1;
    }
}

/* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
   DESTMEM to align it to DESIRED_ALIGNMENT.  Original alignment is ALIGN.
   Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
   ignored.
   Return value is updated DESTMEM.  */
static rtx
expand_set_or_movmem_prologue (rtx destmem, rtx srcmem,
				  rtx destptr, rtx srcptr, rtx value,
				  rtx vec_value, rtx count, int align,
				  int desired_alignment, bool issetmem)
{
  int i;
  for (i = 1; i < desired_alignment; i <<= 1)
    {
      if (align <= i)
	{
	  rtx_code_label *label = ix86_expand_aligntest (destptr, i, false);
	  if (issetmem)
	    {
	      if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
		destmem = emit_memset (destmem, destptr, vec_value, i);
	      else
		destmem = emit_memset (destmem, destptr, value, i);
	    }
	  else
	    destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
	  ix86_adjust_counter (count, i);
	  emit_label (label);
	  LABEL_NUSES (label) = 1;
	  set_mem_align (destmem, i * 2 * BITS_PER_UNIT);
	}
    }
  return destmem;
}

/* Test if COUNT&SIZE is nonzero and if so, expand movme
   or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
   and jump to DONE_LABEL.  */
static void
expand_small_movmem_or_setmem (rtx destmem, rtx srcmem,
			       rtx destptr, rtx srcptr,
			       rtx value, rtx vec_value,
			       rtx count, int size,
			       rtx done_label, bool issetmem)
{
  rtx_code_label *label = ix86_expand_aligntest (count, size, false);
  machine_mode mode = mode_for_size (size * BITS_PER_UNIT, MODE_INT, 1);
  rtx modesize;
  int n;

  /* If we do not have vector value to copy, we must reduce size.  */
  if (issetmem)
    {
      if (!vec_value)
	{
	  if (GET_MODE (value) == VOIDmode && size > 8)
	    mode = Pmode;
	  else if (GET_MODE_SIZE (mode) > GET_MODE_SIZE (GET_MODE (value)))
	    mode = GET_MODE (value);
	}
      else
	mode = GET_MODE (vec_value), value = vec_value;
    }
  else
    {
      /* Choose appropriate vector mode.  */
      if (size >= 32)
	mode = TARGET_AVX ? V32QImode : TARGET_SSE ? V16QImode : DImode;
      else if (size >= 16)
	mode = TARGET_SSE ? V16QImode : DImode;
      srcmem = change_address (srcmem, mode, srcptr);
    }
  destmem = change_address (destmem, mode, destptr);
  modesize = GEN_INT (GET_MODE_SIZE (mode));
  gcc_assert (GET_MODE_SIZE (mode) <= size);
  for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
    {
      if (issetmem)
	emit_move_insn (destmem, gen_lowpart (mode, value));
      else
	{
          emit_move_insn (destmem, srcmem);
          srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
	}
      destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
    }

  destmem = offset_address (destmem, count, 1);
  destmem = offset_address (destmem, GEN_INT (-2 * size),
			    GET_MODE_SIZE (mode));
  if (!issetmem)
    {
      srcmem = offset_address (srcmem, count, 1);
      srcmem = offset_address (srcmem, GEN_INT (-2 * size),
			       GET_MODE_SIZE (mode));
    }
  for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
    {
      if (issetmem)
	emit_move_insn (destmem, gen_lowpart (mode, value));
      else
	{
	  emit_move_insn (destmem, srcmem);
	  srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
	}
      destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
    }
  emit_jump_insn (gen_jump (done_label));
  emit_barrier ();

  emit_label (label);
  LABEL_NUSES (label) = 1;
}

/* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
   and get ready for the main memcpy loop by copying iniital DESIRED_ALIGN-ALIGN
   bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
   proceed with an loop copying SIZE bytes at once. Do moves in MODE.
   DONE_LABEL is a label after the whole copying sequence. The label is created
   on demand if *DONE_LABEL is NULL.
   MIN_SIZE is minimal size of block copied.  This value gets adjusted for new
   bounds after the initial copies. 

   DESTMEM/SRCMEM are memory expressions pointing to the copies block,
   DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
   we will dispatch to a library call for large blocks.

   In pseudocode we do:

   if (COUNT < SIZE)
     {
       Assume that SIZE is 4. Bigger sizes are handled analogously
       if (COUNT & 4)
	 {
	    copy 4 bytes from SRCPTR to DESTPTR
	    copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
	    goto done_label
	 }
       if (!COUNT)
	 goto done_label;
       copy 1 byte from SRCPTR to DESTPTR
       if (COUNT & 2)
	 {
	    copy 2 bytes from SRCPTR to DESTPTR
	    copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
	 }
     }
   else
     {
       copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
       copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE

       OLD_DESPTR = DESTPTR;
       Align DESTPTR up to DESIRED_ALIGN
       SRCPTR += DESTPTR - OLD_DESTPTR
       COUNT -= DEST_PTR - OLD_DESTPTR
       if (DYNAMIC_CHECK)
	 Round COUNT down to multiple of SIZE
       << optional caller supplied zero size guard is here >>
       << optional caller supplied dynamic check is here >>
       << caller supplied main copy loop is here >>
     }
   done_label:
  */
static void
expand_set_or_movmem_prologue_epilogue_by_misaligned_moves (rtx destmem, rtx srcmem,
							    rtx *destptr, rtx *srcptr,
							    machine_mode mode,
							    rtx value, rtx vec_value,
							    rtx *count,
							    rtx_code_label **done_label,
							    int size,
							    int desired_align,
							    int align,
							    unsigned HOST_WIDE_INT *min_size,
							    bool dynamic_check,
							    bool issetmem)
{
  rtx_code_label *loop_label = NULL, *label;
  int n;
  rtx modesize;
  int prolog_size = 0;
  rtx mode_value;

  /* Chose proper value to copy.  */
  if (issetmem && VECTOR_MODE_P (mode))
    mode_value = vec_value;
  else
    mode_value = value;
  gcc_assert (GET_MODE_SIZE (mode) <= size);

  /* See if block is big or small, handle small blocks.  */
  if (!CONST_INT_P (*count) && *min_size < (unsigned HOST_WIDE_INT)size)
    {
      int size2 = size;
      loop_label = gen_label_rtx ();

      if (!*done_label)
	*done_label = gen_label_rtx ();

      emit_cmp_and_jump_insns (*count, GEN_INT (size2), GE, 0, GET_MODE (*count),
			       1, loop_label);
      size2 >>= 1;

      /* Handle sizes > 3.  */
      for (;size2 > 2; size2 >>= 1)
	expand_small_movmem_or_setmem (destmem, srcmem,
				       *destptr, *srcptr,
				       value, vec_value,
				       *count,
				       size2, *done_label, issetmem);
      /* Nothing to copy?  Jump to DONE_LABEL if so */
      emit_cmp_and_jump_insns (*count, const0_rtx, EQ, 0, GET_MODE (*count),
			       1, *done_label);

      /* Do a byte copy.  */
      destmem = change_address (destmem, QImode, *destptr);
      if (issetmem)
	emit_move_insn (destmem, gen_lowpart (QImode, value));
      else
	{
          srcmem = change_address (srcmem, QImode, *srcptr);
          emit_move_insn (destmem, srcmem);
	}

      /* Handle sizes 2 and 3.  */
      label = ix86_expand_aligntest (*count, 2, false);
      destmem = change_address (destmem, HImode, *destptr);
      destmem = offset_address (destmem, *count, 1);
      destmem = offset_address (destmem, GEN_INT (-2), 2);
      if (issetmem)
        emit_move_insn (destmem, gen_lowpart (HImode, value));
      else
	{
	  srcmem = change_address (srcmem, HImode, *srcptr);
	  srcmem = offset_address (srcmem, *count, 1);
	  srcmem = offset_address (srcmem, GEN_INT (-2), 2);
	  emit_move_insn (destmem, srcmem);
	}

      emit_label (label);
      LABEL_NUSES (label) = 1;
      emit_jump_insn (gen_jump (*done_label));
      emit_barrier ();
    }
  else
    gcc_assert (*min_size >= (unsigned HOST_WIDE_INT)size
		|| UINTVAL (*count) >= (unsigned HOST_WIDE_INT)size);

  /* Start memcpy for COUNT >= SIZE.  */
  if (loop_label)
    {
       emit_label (loop_label);
       LABEL_NUSES (loop_label) = 1;
    }

  /* Copy first desired_align bytes.  */
  if (!issetmem)
    srcmem = change_address (srcmem, mode, *srcptr);
  destmem = change_address (destmem, mode, *destptr);
  modesize = GEN_INT (GET_MODE_SIZE (mode));
  for (n = 0; prolog_size < desired_align - align; n++)
    {
      if (issetmem)
        emit_move_insn (destmem, mode_value);
      else
	{
          emit_move_insn (destmem, srcmem);
          srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
	}
      destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
      prolog_size += GET_MODE_SIZE (mode);
    }


  /* Copy last SIZE bytes.  */
  destmem = offset_address (destmem, *count, 1);
  destmem = offset_address (destmem,
			    GEN_INT (-size - prolog_size),
			    1);
  if (issetmem)
    emit_move_insn (destmem, mode_value);
  else
    {
      srcmem = offset_address (srcmem, *count, 1);
      srcmem = offset_address (srcmem,
			       GEN_INT (-size - prolog_size),
			       1);
      emit_move_insn (destmem, srcmem);
    }
  for (n = 1; n * GET_MODE_SIZE (mode) < size; n++)
    {
      destmem = offset_address (destmem, modesize, 1);
      if (issetmem)
	emit_move_insn (destmem, mode_value);
      else
	{
          srcmem = offset_address (srcmem, modesize, 1);
          emit_move_insn (destmem, srcmem);
	}
    }

  /* Align destination.  */
  if (desired_align > 1 && desired_align > align)
    {
      rtx saveddest = *destptr;

      gcc_assert (desired_align <= size);
      /* Align destptr up, place it to new register.  */
      *destptr = expand_simple_binop (GET_MODE (*destptr), PLUS, *destptr,
				      GEN_INT (prolog_size),
				      NULL_RTX, 1, OPTAB_DIRECT);
      if (REG_P (*destptr) && REG_P (saveddest) && REG_POINTER (saveddest))
	REG_POINTER (*destptr) = 1;
      *destptr = expand_simple_binop (GET_MODE (*destptr), AND, *destptr,
				      GEN_INT (-desired_align),
				      *destptr, 1, OPTAB_DIRECT);
      /* See how many bytes we skipped.  */
      saveddest = expand_simple_binop (GET_MODE (*destptr), MINUS, saveddest,
				       *destptr,
				       saveddest, 1, OPTAB_DIRECT);
      /* Adjust srcptr and count.  */
      if (!issetmem)
	*srcptr = expand_simple_binop (GET_MODE (*srcptr), MINUS, *srcptr,
				       saveddest, *srcptr, 1, OPTAB_DIRECT);
      *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
				    saveddest, *count, 1, OPTAB_DIRECT);
      /* We copied at most size + prolog_size.  */
      if (*min_size > (unsigned HOST_WIDE_INT)(size + prolog_size))
	*min_size
	  = ROUND_DOWN (*min_size - size, (unsigned HOST_WIDE_INT)size);
      else
	*min_size = 0;

      /* Our loops always round down the block size, but for dispatch to
         library we need precise value.  */
      if (dynamic_check)
	*count = expand_simple_binop (GET_MODE (*count), AND, *count,
				      GEN_INT (-size), *count, 1, OPTAB_DIRECT);
    }
  else
    {
      gcc_assert (prolog_size == 0);
      /* Decrease count, so we won't end up copying last word twice.  */
      if (!CONST_INT_P (*count))
	*count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
				      constm1_rtx, *count, 1, OPTAB_DIRECT);
      else
	*count = GEN_INT (ROUND_DOWN (UINTVAL (*count) - 1,
				      (unsigned HOST_WIDE_INT)size));
      if (*min_size)
	*min_size = ROUND_DOWN (*min_size - 1, (unsigned HOST_WIDE_INT)size);
    }
}


/* This function is like the previous one, except here we know how many bytes
   need to be copied.  That allows us to update alignment not only of DST, which
   is returned, but also of SRC, which is passed as a pointer for that
   reason.  */
static rtx
expand_set_or_movmem_constant_prologue (rtx dst, rtx *srcp, rtx destreg,
					   rtx srcreg, rtx value, rtx vec_value,
					   int desired_align, int align_bytes,
					   bool issetmem)
{
  rtx src = NULL;
  rtx orig_dst = dst;
  rtx orig_src = NULL;
  int piece_size = 1;
  int copied_bytes = 0;

  if (!issetmem)
    {
      gcc_assert (srcp != NULL);
      src = *srcp;
      orig_src = src;
    }

  for (piece_size = 1;
       piece_size <= desired_align && copied_bytes < align_bytes;
       piece_size <<= 1)
    {
      if (align_bytes & piece_size)
	{
	  if (issetmem)
	    {
	      if (vec_value && piece_size > GET_MODE_SIZE (GET_MODE (value)))
		dst = emit_memset (dst, destreg, vec_value, piece_size);
	      else
		dst = emit_memset (dst, destreg, value, piece_size);
	    }
	  else
	    dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
	  copied_bytes += piece_size;
	}
    }
  if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
    set_mem_align (dst, desired_align * BITS_PER_UNIT);
  if (MEM_SIZE_KNOWN_P (orig_dst))
    set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);

  if (!issetmem)
    {
      int src_align_bytes = get_mem_align_offset (src, desired_align
						       * BITS_PER_UNIT);
      if (src_align_bytes >= 0)
	src_align_bytes = desired_align - src_align_bytes;
      if (src_align_bytes >= 0)
	{
	  unsigned int src_align;
	  for (src_align = desired_align; src_align >= 2; src_align >>= 1)
	    {
	      if ((src_align_bytes & (src_align - 1))
		   == (align_bytes & (src_align - 1)))
		break;
	    }
	  if (src_align > (unsigned int) desired_align)
	    src_align = desired_align;
	  if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
	    set_mem_align (src, src_align * BITS_PER_UNIT);
	}
      if (MEM_SIZE_KNOWN_P (orig_src))
	set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
      *srcp = src;
    }

  return dst;
}

/* Return true if ALG can be used in current context.  
   Assume we expand memset if MEMSET is true.  */
static bool
alg_usable_p (enum stringop_alg alg, bool memset, bool have_as)
{
  if (alg == no_stringop)
    return false;
  if (alg == vector_loop)
    return TARGET_SSE || TARGET_AVX;
  /* Algorithms using the rep prefix want at least edi and ecx;
     additionally, memset wants eax and memcpy wants esi.  Don't
     consider such algorithms if the user has appropriated those
     registers for their own purposes, or if we have a non-default
     address space, since some string insns cannot override the segment.  */
  if (alg == rep_prefix_1_byte
      || alg == rep_prefix_4_byte
      || alg == rep_prefix_8_byte)
    {
      if (have_as)
	return false;
      if (fixed_regs[CX_REG]
	  || fixed_regs[DI_REG]
	  || (memset ? fixed_regs[AX_REG] : fixed_regs[SI_REG]))
	return false;
    }
  return true;
}

/* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation.  */
static enum stringop_alg
decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size,
	    unsigned HOST_WIDE_INT min_size, unsigned HOST_WIDE_INT max_size,
	    bool memset, bool zero_memset, bool have_as,
	    int *dynamic_check, bool *noalign, bool recur)
{
  const struct stringop_algs *algs;
  bool optimize_for_speed;
  int max = 0;
  const struct processor_costs *cost;
  int i;
  bool any_alg_usable_p = false;

  *noalign = false;
  *dynamic_check = -1;

  /* Even if the string operation call is cold, we still might spend a lot
     of time processing large blocks.  */
  if (optimize_function_for_size_p (cfun)
      || (optimize_insn_for_size_p ()
 	  && (max_size < 256
              || (expected_size != -1 && expected_size < 256))))
    optimize_for_speed = false;
  else
    optimize_for_speed = true;

  cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
  if (memset)
    algs = &cost->memset[TARGET_64BIT != 0];
  else
    algs = &cost->memcpy[TARGET_64BIT != 0];

  /* See maximal size for user defined algorithm.  */
  for (i = 0; i < MAX_STRINGOP_ALGS; i++)
    {
      enum stringop_alg candidate = algs->size[i].alg;
      bool usable = alg_usable_p (candidate, memset, have_as);
      any_alg_usable_p |= usable;

      if (candidate != libcall && candidate && usable)
	max = algs->size[i].max;
    }

  /* If expected size is not known but max size is small enough
     so inline version is a win, set expected size into
     the range.  */
  if (((max > 1 && (unsigned HOST_WIDE_INT) max >= max_size) || max == -1)
      && expected_size == -1)
    expected_size = min_size / 2 + max_size / 2;

  /* If user specified the algorithm, honor it if possible.  */
  if (ix86_stringop_alg != no_stringop
      && alg_usable_p (ix86_stringop_alg, memset, have_as))
    return ix86_stringop_alg;
  /* rep; movq or rep; movl is the smallest variant.  */
  else if (!optimize_for_speed)
    {
      *noalign = true;
      if (!count || (count & 3) || (memset && !zero_memset))
	return alg_usable_p (rep_prefix_1_byte, memset, have_as)
	       ? rep_prefix_1_byte : loop_1_byte;
      else
	return alg_usable_p (rep_prefix_4_byte, memset, have_as)
	       ? rep_prefix_4_byte : loop;
    }
  /* Very tiny blocks are best handled via the loop, REP is expensive to
     setup.  */
  else if (expected_size != -1 && expected_size < 4)
    return loop_1_byte;
  else if (expected_size != -1)
    {
      enum stringop_alg alg = libcall;
      bool alg_noalign = false;
      for (i = 0; i < MAX_STRINGOP_ALGS; i++)
	{
	  /* We get here if the algorithms that were not libcall-based
	     were rep-prefix based and we are unable to use rep prefixes
	     based on global register usage.  Break out of the loop and
	     use the heuristic below.  */
	  if (algs->size[i].max == 0)
	    break;
	  if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
	    {
	      enum stringop_alg candidate = algs->size[i].alg;

	      if (candidate != libcall
		  && alg_usable_p (candidate, memset, have_as))
		{
		  alg = candidate;
		  alg_noalign = algs->size[i].noalign;
		}
	      /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
		 last non-libcall inline algorithm.  */
	      if (TARGET_INLINE_ALL_STRINGOPS)
		{
		  /* When the current size is best to be copied by a libcall,
		     but we are still forced to inline, run the heuristic below
		     that will pick code for medium sized blocks.  */
		  if (alg != libcall)
		    {
		      *noalign = alg_noalign;
		      return alg;
		    }
		  else if (!any_alg_usable_p)
		    break;
		}
	      else if (alg_usable_p (candidate, memset, have_as))
		{
		  *noalign = algs->size[i].noalign;
		  return candidate;
		}
	    }
	}
    }
  /* When asked to inline the call anyway, try to pick meaningful choice.
     We look for maximal size of block that is faster to copy by hand and
     take blocks of at most of that size guessing that average size will
     be roughly half of the block.

     If this turns out to be bad, we might simply specify the preferred
     choice in ix86_costs.  */
  if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
      && (algs->unknown_size == libcall
	  || !alg_usable_p (algs->unknown_size, memset, have_as)))
    {
      enum stringop_alg alg;
      HOST_WIDE_INT new_expected_size = (max > 0 ? max : 4096) / 2;

      /* If there aren't any usable algorithms or if recursing already,
	 then recursing on smaller sizes or same size isn't going to
	 find anything.  Just return the simple byte-at-a-time copy loop.  */
      if (!any_alg_usable_p || recur)
	{
	  /* Pick something reasonable.  */
	  if (TARGET_INLINE_STRINGOPS_DYNAMICALLY && !recur)
	    *dynamic_check = 128;
	  return loop_1_byte;
	}
      alg = decide_alg (count, new_expected_size, min_size, max_size, memset,
			zero_memset, have_as, dynamic_check, noalign, true);
      gcc_assert (*dynamic_check == -1);
      if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
	*dynamic_check = max;
      else
	gcc_assert (alg != libcall);
      return alg;
    }
  return (alg_usable_p (algs->unknown_size, memset, have_as)
	  ? algs->unknown_size : libcall);
}

/* Decide on alignment.  We know that the operand is already aligned to ALIGN
   (ALIGN can be based on profile feedback and thus it is not 100% guaranteed).  */
static int
decide_alignment (int align,
		  enum stringop_alg alg,
		  int expected_size,
		  machine_mode move_mode)
{
  int desired_align = 0;

  gcc_assert (alg != no_stringop);

  if (alg == libcall)
    return 0;
  if (move_mode == VOIDmode)
    return 0;

  desired_align = GET_MODE_SIZE (move_mode);
  /* PentiumPro has special logic triggering for 8 byte aligned blocks.
     copying whole cacheline at once.  */
  if (TARGET_PENTIUMPRO
      && (alg == rep_prefix_4_byte || alg == rep_prefix_1_byte))
    desired_align = 8;

  if (optimize_size)
    desired_align = 1;
  if (desired_align < align)
    desired_align = align;
  if (expected_size != -1 && expected_size < 4)
    desired_align = align;

  return desired_align;
}


/* Helper function for memcpy.  For QImode value 0xXY produce
   0xXYXYXYXY of wide specified by MODE.  This is essentially
   a * 0x10101010, but we can do slightly better than
   synth_mult by unwinding the sequence by hand on CPUs with
   slow multiply.  */
static rtx
promote_duplicated_reg (machine_mode mode, rtx val)
{
  machine_mode valmode = GET_MODE (val);
  rtx tmp;
  int nops = mode == DImode ? 3 : 2;

  gcc_assert (mode == SImode || mode == DImode || val == const0_rtx);
  if (val == const0_rtx)
    return copy_to_mode_reg (mode, CONST0_RTX (mode));
  if (CONST_INT_P (val))
    {
      HOST_WIDE_INT v = INTVAL (val) & 255;

      v |= v << 8;
      v |= v << 16;
      if (mode == DImode)
        v |= (v << 16) << 16;
      return copy_to_mode_reg (mode, gen_int_mode (v, mode));
    }

  if (valmode == VOIDmode)
    valmode = QImode;
  if (valmode != QImode)
    val = gen_lowpart (QImode, val);
  if (mode == QImode)
    return val;
  if (!TARGET_PARTIAL_REG_STALL)
    nops--;
  if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
      + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
      <= (ix86_cost->shift_const + ix86_cost->add) * nops
          + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
    {
      rtx reg = convert_modes (mode, QImode, val, true);
      tmp = promote_duplicated_reg (mode, const1_rtx);
      return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
				  OPTAB_DIRECT);
    }
  else
    {
      rtx reg = convert_modes (mode, QImode, val, true);

      if (!TARGET_PARTIAL_REG_STALL)
	if (mode == SImode)
	  emit_insn (gen_insvsi_1 (reg, reg));
	else
	  emit_insn (gen_insvdi_1 (reg, reg));
      else
	{
	  tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
				     NULL, 1, OPTAB_DIRECT);
	  reg =
	    expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
	}
      tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
			         NULL, 1, OPTAB_DIRECT);
      reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
      if (mode == SImode)
	return reg;
      tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
				 NULL, 1, OPTAB_DIRECT);
      reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
      return reg;
    }
}

/* Duplicate value VAL using promote_duplicated_reg into maximal size that will
   be needed by main loop copying SIZE_NEEDED chunks and prologue getting
   alignment from ALIGN to DESIRED_ALIGN.  */
static rtx
promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align,
				int align)
{
  rtx promoted_val;

  if (TARGET_64BIT
      && (size_needed > 4 || (desired_align > align && desired_align > 4)))
    promoted_val = promote_duplicated_reg (DImode, val);
  else if (size_needed > 2 || (desired_align > align && desired_align > 2))
    promoted_val = promote_duplicated_reg (SImode, val);
  else if (size_needed > 1 || (desired_align > align && desired_align > 1))
    promoted_val = promote_duplicated_reg (HImode, val);
  else
    promoted_val = val;

  return promoted_val;
}

/* Expand string move (memcpy) ot store (memset) operation.  Use i386 string
   operations when profitable.  The code depends upon architecture, block size
   and alignment, but always has one of the following overall structures:

   Aligned move sequence:

     1) Prologue guard: Conditional that jumps up to epilogues for small
	blocks that can be handled by epilogue alone.  This is faster
	but also needed for correctness, since prologue assume the block
	is larger than the desired alignment.

	Optional dynamic check for size and libcall for large
	blocks is emitted here too, with -minline-stringops-dynamically.

     2) Prologue: copy first few bytes in order to get destination
	aligned to DESIRED_ALIGN.  It is emitted only when ALIGN is less
	than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
	copied.  We emit either a jump tree on power of two sized
	blocks, or a byte loop.

     3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
	with specified algorithm.

     4) Epilogue: code copying tail of the block that is too small to be
	handled by main body (or up to size guarded by prologue guard). 

  Misaligned move sequence

     1) missaligned move prologue/epilogue containing:
        a) Prologue handling small memory blocks and jumping to done_label
	   (skipped if blocks are known to be large enough)
	b) Signle move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
           needed by single possibly misaligned move
	   (skipped if alignment is not needed)
        c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves

     2) Zero size guard dispatching to done_label, if needed

     3) dispatch to library call, if needed,

     3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
	with specified algorithm.  */
bool
ix86_expand_set_or_movmem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
			   rtx align_exp, rtx expected_align_exp,
			   rtx expected_size_exp, rtx min_size_exp,
			   rtx max_size_exp, rtx probable_max_size_exp,
			   bool issetmem)
{
  rtx destreg;
  rtx srcreg = NULL;
  rtx_code_label *label = NULL;
  rtx tmp;
  rtx_code_label *jump_around_label = NULL;
  HOST_WIDE_INT align = 1;
  unsigned HOST_WIDE_INT count = 0;
  HOST_WIDE_INT expected_size = -1;
  int size_needed = 0, epilogue_size_needed;
  int desired_align = 0, align_bytes = 0;
  enum stringop_alg alg;
  rtx promoted_val = NULL;
  rtx vec_promoted_val = NULL;
  bool force_loopy_epilogue = false;
  int dynamic_check;
  bool need_zero_guard = false;
  bool noalign;
  machine_mode move_mode = VOIDmode;
  int unroll_factor = 1;
  /* TODO: Once value ranges are available, fill in proper data.  */
  unsigned HOST_WIDE_INT min_size = 0;
  unsigned HOST_WIDE_INT max_size = -1;
  unsigned HOST_WIDE_INT probable_max_size = -1;
  bool misaligned_prologue_used = false;
  bool have_as;

  if (CONST_INT_P (align_exp))
    align = INTVAL (align_exp);
  /* i386 can do misaligned access on reasonably increased cost.  */
  if (CONST_INT_P (expected_align_exp)
      && INTVAL (expected_align_exp) > align)
    align = INTVAL (expected_align_exp);
  /* ALIGN is the minimum of destination and source alignment, but we care here
     just about destination alignment.  */
  else if (!issetmem
	   && MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
    align = MEM_ALIGN (dst) / BITS_PER_UNIT;

  if (CONST_INT_P (count_exp))
    {
      min_size = max_size = probable_max_size = count = expected_size
	= INTVAL (count_exp);
      /* When COUNT is 0, there is nothing to do.  */
      if (!count)
	return true;
    }
  else
    {
      if (min_size_exp)
	min_size = INTVAL (min_size_exp);
      if (max_size_exp)
	max_size = INTVAL (max_size_exp);
      if (probable_max_size_exp)
	probable_max_size = INTVAL (probable_max_size_exp);
      if (CONST_INT_P (expected_size_exp))
	expected_size = INTVAL (expected_size_exp);
     }

  /* Make sure we don't need to care about overflow later on.  */
  if (count > (HOST_WIDE_INT_1U << 30))
    return false;

  have_as = !ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (dst));
  if (!issetmem)
    have_as |= !ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (src));

  /* Step 0: Decide on preferred algorithm, desired alignment and
     size of chunks to be copied by main loop.  */
  alg = decide_alg (count, expected_size, min_size, probable_max_size,
		    issetmem,
		    issetmem && val_exp == const0_rtx, have_as,
		    &dynamic_check, &noalign, false);
  if (alg == libcall)
    return false;
  gcc_assert (alg != no_stringop);

  /* For now vector-version of memset is generated only for memory zeroing, as
     creating of promoted vector value is very cheap in this case.  */
  if (issetmem && alg == vector_loop && val_exp != const0_rtx)
    alg = unrolled_loop;

  if (!count)
    count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
  destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
  if (!issetmem)
    srcreg = ix86_copy_addr_to_reg (XEXP (src, 0));

  unroll_factor = 1;
  move_mode = word_mode;
  switch (alg)
    {
    case libcall:
    case no_stringop:
    case last_alg:
      gcc_unreachable ();
    case loop_1_byte:
      need_zero_guard = true;
      move_mode = QImode;
      break;
    case loop:
      need_zero_guard = true;
      break;
    case unrolled_loop:
      need_zero_guard = true;
      unroll_factor = (TARGET_64BIT ? 4 : 2);
      break;
    case vector_loop:
      need_zero_guard = true;
      unroll_factor = 4;
      /* Find the widest supported mode.  */
      move_mode = word_mode;
      while (optab_handler (mov_optab, GET_MODE_WIDER_MODE (move_mode))
	     != CODE_FOR_nothing)
	  move_mode = GET_MODE_WIDER_MODE (move_mode);

      /* Find the corresponding vector mode with the same size as MOVE_MODE.
	 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.).  */
      if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
	{
	  int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
	  move_mode = mode_for_vector (word_mode, nunits);
	  if (optab_handler (mov_optab, move_mode) == CODE_FOR_nothing)
	    move_mode = word_mode;
	}
      gcc_assert (optab_handler (mov_optab, move_mode) != CODE_FOR_nothing);
      break;
    case rep_prefix_8_byte:
      move_mode = DImode;
      break;
    case rep_prefix_4_byte:
      move_mode = SImode;
      break;
    case rep_prefix_1_byte:
      move_mode = QImode;
      break;
    }
  size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
  epilogue_size_needed = size_needed;

  /* If we are going to call any library calls conditionally, make sure any
     pending stack adjustment happen before the first conditional branch,
     otherwise they will be emitted before the library call only and won't
     happen from the other branches.  */
  if (dynamic_check != -1)
    do_pending_stack_adjust ();

  desired_align = decide_alignment (align, alg, expected_size, move_mode);
  if (!TARGET_ALIGN_STRINGOPS || noalign)
    align = desired_align;

  /* Step 1: Prologue guard.  */

  /* Alignment code needs count to be in register.  */
  if (CONST_INT_P (count_exp) && desired_align > align)
    {
      if (INTVAL (count_exp) > desired_align
	  && INTVAL (count_exp) > size_needed)
	{
	  align_bytes
	    = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
	  if (align_bytes <= 0)
	    align_bytes = 0;
	  else
	    align_bytes = desired_align - align_bytes;
	}
      if (align_bytes == 0)
	count_exp = force_reg (counter_mode (count_exp), count_exp);
    }
  gcc_assert (desired_align >= 1 && align >= 1);

  /* Misaligned move sequences handle both prologue and epilogue at once.
     Default code generation results in a smaller code for large alignments
     and also avoids redundant job when sizes are known precisely.  */
  misaligned_prologue_used
    = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
       && MAX (desired_align, epilogue_size_needed) <= 32
       && desired_align <= epilogue_size_needed
       && ((desired_align > align && !align_bytes)
	   || (!count && epilogue_size_needed > 1)));

  /* Do the cheap promotion to allow better CSE across the
     main loop and epilogue (ie one load of the big constant in the
     front of all code.  
     For now the misaligned move sequences do not have fast path
     without broadcasting.  */
  if (issetmem && ((CONST_INT_P (val_exp) || misaligned_prologue_used)))
    {
      if (alg == vector_loop)
	{
	  gcc_assert (val_exp == const0_rtx);
	  vec_promoted_val = promote_duplicated_reg (move_mode, val_exp);
	  promoted_val = promote_duplicated_reg_to_size (val_exp,
							 GET_MODE_SIZE (word_mode),
							 desired_align, align);
	}
      else
	{
	  promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
							 desired_align, align);
	}
    }
  /* Misaligned move sequences handles both prologues and epilogues at once.
     Default code generation results in smaller code for large alignments and
     also avoids redundant job when sizes are known precisely.  */
  if (misaligned_prologue_used)
    {
      /* Misaligned move prologue handled small blocks by itself.  */
      expand_set_or_movmem_prologue_epilogue_by_misaligned_moves
	   (dst, src, &destreg, &srcreg,
	    move_mode, promoted_val, vec_promoted_val,
	    &count_exp,
	    &jump_around_label,
            desired_align < align
	    ? MAX (desired_align, epilogue_size_needed) : epilogue_size_needed,
	    desired_align, align, &min_size, dynamic_check, issetmem);
      if (!issetmem)
        src = change_address (src, BLKmode, srcreg);
      dst = change_address (dst, BLKmode, destreg);
      set_mem_align (dst, desired_align * BITS_PER_UNIT);
      epilogue_size_needed = 0;
      if (need_zero_guard
	  && min_size < (unsigned HOST_WIDE_INT) size_needed)
	{
	  /* It is possible that we copied enough so the main loop will not
	     execute.  */
	  gcc_assert (size_needed > 1);
	  if (jump_around_label == NULL_RTX)
	    jump_around_label = gen_label_rtx ();
	  emit_cmp_and_jump_insns (count_exp,
				   GEN_INT (size_needed),
				   LTU, 0, counter_mode (count_exp), 1, jump_around_label);
	  if (expected_size == -1
	      || expected_size < (desired_align - align) / 2 + size_needed)
	    predict_jump (REG_BR_PROB_BASE * 20 / 100);
	  else
	    predict_jump (REG_BR_PROB_BASE * 60 / 100);
	}
    }
  /* Ensure that alignment prologue won't copy past end of block.  */
  else if (size_needed > 1 || (desired_align > 1 && desired_align > align))
    {
      epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
      /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
	 Make sure it is power of 2.  */
      epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);

      /* To improve performance of small blocks, we jump around the VAL
	 promoting mode.  This mean that if the promoted VAL is not constant,
	 we might not use it in the epilogue and have to use byte
	 loop variant.  */
      if (issetmem && epilogue_size_needed > 2 && !promoted_val)
	force_loopy_epilogue = true;
      if ((count && count < (unsigned HOST_WIDE_INT) epilogue_size_needed)
	  || max_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
	{
	  /* If main algorithm works on QImode, no epilogue is needed.
	     For small sizes just don't align anything.  */
	  if (size_needed == 1)
	    desired_align = align;
	  else
	    goto epilogue;
	}
      else if (!count
	       && min_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
	{
	  label = gen_label_rtx ();
	  emit_cmp_and_jump_insns (count_exp,
				   GEN_INT (epilogue_size_needed),
				   LTU, 0, counter_mode (count_exp), 1, label);
	  if (expected_size == -1 || expected_size < epilogue_size_needed)
	    predict_jump (REG_BR_PROB_BASE * 60 / 100);
	  else
	    predict_jump (REG_BR_PROB_BASE * 20 / 100);
	}
    }

  /* Emit code to decide on runtime whether library call or inline should be
     used.  */
  if (dynamic_check != -1)
    {
      if (!issetmem && CONST_INT_P (count_exp))
	{
	  if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
	    {
	      emit_block_copy_via_libcall (dst, src, count_exp);
	      count_exp = const0_rtx;
	      goto epilogue;
	    }
	}
      else
	{
	  rtx_code_label *hot_label = gen_label_rtx ();
	  if (jump_around_label == NULL_RTX)
	    jump_around_label = gen_label_rtx ();
	  emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
				   LEU, 0, counter_mode (count_exp),
				   1, hot_label);
	  predict_jump (REG_BR_PROB_BASE * 90 / 100);
	  if (issetmem)
	    set_storage_via_libcall (dst, count_exp, val_exp);
	  else
	    emit_block_copy_via_libcall (dst, src, count_exp);
	  emit_jump (jump_around_label);
	  emit_label (hot_label);
	}
    }

  /* Step 2: Alignment prologue.  */
  /* Do the expensive promotion once we branched off the small blocks.  */
  if (issetmem && !promoted_val)
    promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
						   desired_align, align);

  if (desired_align > align && !misaligned_prologue_used)
    {
      if (align_bytes == 0)
	{
	  /* Except for the first move in prologue, we no longer know
	     constant offset in aliasing info.  It don't seems to worth
	     the pain to maintain it for the first move, so throw away
	     the info early.  */
	  dst = change_address (dst, BLKmode, destreg);
	  if (!issetmem)
	    src = change_address (src, BLKmode, srcreg);
	  dst = expand_set_or_movmem_prologue (dst, src, destreg, srcreg,
					    promoted_val, vec_promoted_val,
					    count_exp, align, desired_align,
					    issetmem);
	  /* At most desired_align - align bytes are copied.  */
	  if (min_size < (unsigned)(desired_align - align))
	    min_size = 0;
	  else
	    min_size -= desired_align - align;
	}
      else
	{
	  /* If we know how many bytes need to be stored before dst is
	     sufficiently aligned, maintain aliasing info accurately.  */
	  dst = expand_set_or_movmem_constant_prologue (dst, &src, destreg,
							   srcreg,
							   promoted_val,
							   vec_promoted_val,
							   desired_align,
							   align_bytes,
							   issetmem);

	  count_exp = plus_constant (counter_mode (count_exp),
				     count_exp, -align_bytes);
	  count -= align_bytes;
	  min_size -= align_bytes;
	  max_size -= align_bytes;
	}
      if (need_zero_guard
	  && min_size < (unsigned HOST_WIDE_INT) size_needed
	  && (count < (unsigned HOST_WIDE_INT) size_needed
	      || (align_bytes == 0
		  && count < ((unsigned HOST_WIDE_INT) size_needed
			      + desired_align - align))))
	{
	  /* It is possible that we copied enough so the main loop will not
	     execute.  */
	  gcc_assert (size_needed > 1);
	  if (label == NULL_RTX)
	    label = gen_label_rtx ();
	  emit_cmp_and_jump_insns (count_exp,
				   GEN_INT (size_needed),
				   LTU, 0, counter_mode (count_exp), 1, label);
	  if (expected_size == -1
	      || expected_size < (desired_align - align) / 2 + size_needed)
	    predict_jump (REG_BR_PROB_BASE * 20 / 100);
	  else
	    predict_jump (REG_BR_PROB_BASE * 60 / 100);
	}
    }
  if (label && size_needed == 1)
    {
      emit_label (label);
      LABEL_NUSES (label) = 1;
      label = NULL;
      epilogue_size_needed = 1;
      if (issetmem)
	promoted_val = val_exp;
    }
  else if (label == NULL_RTX && !misaligned_prologue_used)
    epilogue_size_needed = size_needed;

  /* Step 3: Main loop.  */

  switch (alg)
    {
    case libcall:
    case no_stringop:
    case last_alg:
      gcc_unreachable ();
    case loop_1_byte:
    case loop:
    case unrolled_loop:
      expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, promoted_val,
				     count_exp, move_mode, unroll_factor,
				     expected_size, issetmem);
      break;
    case vector_loop:
      expand_set_or_movmem_via_loop (dst, src, destreg, srcreg,
				     vec_promoted_val, count_exp, move_mode,
				     unroll_factor, expected_size, issetmem);
      break;
    case rep_prefix_8_byte:
    case rep_prefix_4_byte:
    case rep_prefix_1_byte:
      expand_set_or_movmem_via_rep (dst, src, destreg, srcreg, promoted_val,
				       val_exp, count_exp, move_mode, issetmem);
      break;
    }
  /* Adjust properly the offset of src and dest memory for aliasing.  */
  if (CONST_INT_P (count_exp))
    {
      if (!issetmem)
	src = adjust_automodify_address_nv (src, BLKmode, srcreg,
					    (count / size_needed) * size_needed);
      dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
					  (count / size_needed) * size_needed);
    }
  else
    {
      if (!issetmem)
	src = change_address (src, BLKmode, srcreg);
      dst = change_address (dst, BLKmode, destreg);
    }

  /* Step 4: Epilogue to copy the remaining bytes.  */
 epilogue:
  if (label)
    {
      /* When the main loop is done, COUNT_EXP might hold original count,
	 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
	 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
	 bytes. Compensate if needed.  */

      if (size_needed < epilogue_size_needed)
	{
	  tmp =
	    expand_simple_binop (counter_mode (count_exp), AND, count_exp,
				 GEN_INT (size_needed - 1), count_exp, 1,
				 OPTAB_DIRECT);
	  if (tmp != count_exp)
	    emit_move_insn (count_exp, tmp);
	}
      emit_label (label);
      LABEL_NUSES (label) = 1;
    }

  if (count_exp != const0_rtx && epilogue_size_needed > 1)
    {
      if (force_loopy_epilogue)
	expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
					 epilogue_size_needed);
      else
	{
	  if (issetmem)
	    expand_setmem_epilogue (dst, destreg, promoted_val,
				    vec_promoted_val, count_exp,
				    epilogue_size_needed);
	  else
	    expand_movmem_epilogue (dst, src, destreg, srcreg, count_exp,
				    epilogue_size_needed);
	}
    }
  if (jump_around_label)
    emit_label (jump_around_label);
  return true;
}


/* Expand the appropriate insns for doing strlen if not just doing
   repnz; scasb

   out = result, initialized with the start address
   align_rtx = alignment of the address.
   scratch = scratch register, initialized with the startaddress when
	not aligned, otherwise undefined

   This is just the body. It needs the initializations mentioned above and
   some address computing at the end.  These things are done in i386.md.  */

static void
ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
{
  int align;
  rtx tmp;
  rtx_code_label *align_2_label = NULL;
  rtx_code_label *align_3_label = NULL;
  rtx_code_label *align_4_label = gen_label_rtx ();
  rtx_code_label *end_0_label = gen_label_rtx ();
  rtx mem;
  rtx tmpreg = gen_reg_rtx (SImode);
  rtx scratch = gen_reg_rtx (SImode);
  rtx cmp;

  align = 0;
  if (CONST_INT_P (align_rtx))
    align = INTVAL (align_rtx);

  /* Loop to check 1..3 bytes for null to get an aligned pointer.  */

  /* Is there a known alignment and is it less than 4?  */
  if (align < 4)
    {
      rtx scratch1 = gen_reg_rtx (Pmode);
      emit_move_insn (scratch1, out);
      /* Is there a known alignment and is it not 2? */
      if (align != 2)
	{
	  align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
	  align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */

	  /* Leave just the 3 lower bits.  */
	  align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
				    NULL_RTX, 0, OPTAB_WIDEN);

	  emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
				   Pmode, 1, align_4_label);
	  emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
				   Pmode, 1, align_2_label);
	  emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
				   Pmode, 1, align_3_label);
	}
      else
        {
	  /* Since the alignment is 2, we have to check 2 or 0 bytes;
	     check if is aligned to 4 - byte.  */

	  align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
				    NULL_RTX, 0, OPTAB_WIDEN);

	  emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
				   Pmode, 1, align_4_label);
        }

      mem = change_address (src, QImode, out);

      /* Now compare the bytes.  */

      /* Compare the first n unaligned byte on a byte per byte basis.  */
      emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
			       QImode, 1, end_0_label);

      /* Increment the address.  */
      emit_insn (ix86_gen_add3 (out, out, const1_rtx));

      /* Not needed with an alignment of 2 */
      if (align != 2)
	{
	  emit_label (align_2_label);

	  emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
				   end_0_label);

	  emit_insn (ix86_gen_add3 (out, out, const1_rtx));

	  emit_label (align_3_label);
	}

      emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
			       end_0_label);

      emit_insn (ix86_gen_add3 (out, out, const1_rtx));
    }

  /* Generate loop to check 4 bytes at a time.  It is not a good idea to
     align this loop.  It gives only huge programs, but does not help to
     speed up.  */
  emit_label (align_4_label);

  mem = change_address (src, SImode, out);
  emit_move_insn (scratch, mem);
  emit_insn (ix86_gen_add3 (out, out, GEN_INT (4)));

  /* This formula yields a nonzero result iff one of the bytes is zero.
     This saves three branches inside loop and many cycles.  */

  emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
  emit_insn (gen_one_cmplsi2 (scratch, scratch));
  emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
  emit_insn (gen_andsi3 (tmpreg, tmpreg,
			 gen_int_mode (0x80808080, SImode)));
  emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
			   align_4_label);

  if (TARGET_CMOVE)
    {
       rtx reg = gen_reg_rtx (SImode);
       rtx reg2 = gen_reg_rtx (Pmode);
       emit_move_insn (reg, tmpreg);
       emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));

       /* If zero is not in the first two bytes, move two bytes forward.  */
       emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
       tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
       tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
       emit_insn (gen_rtx_SET (tmpreg,
			       gen_rtx_IF_THEN_ELSE (SImode, tmp,
						     reg,
						     tmpreg)));
       /* Emit lea manually to avoid clobbering of flags.  */
       emit_insn (gen_rtx_SET (reg2, gen_rtx_PLUS (Pmode, out, const2_rtx)));

       tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
       tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
       emit_insn (gen_rtx_SET (out,
			       gen_rtx_IF_THEN_ELSE (Pmode, tmp,
						     reg2,
						     out)));
    }
  else
    {
       rtx_code_label *end_2_label = gen_label_rtx ();
       /* Is zero in the first two bytes? */

       emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
       tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
       tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
       tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
                            gen_rtx_LABEL_REF (VOIDmode, end_2_label),
                            pc_rtx);
       tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
       JUMP_LABEL (tmp) = end_2_label;

       /* Not in the first two.  Move two bytes forward.  */
       emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
       emit_insn (ix86_gen_add3 (out, out, const2_rtx));

       emit_label (end_2_label);

    }

  /* Avoid branch in fixing the byte.  */
  tmpreg = gen_lowpart (QImode, tmpreg);
  emit_insn (gen_addqi3_cconly_overflow (tmpreg, tmpreg));
  tmp = gen_rtx_REG (CCmode, FLAGS_REG);
  cmp = gen_rtx_LTU (VOIDmode, tmp, const0_rtx);
  emit_insn (ix86_gen_sub3_carry (out, out, GEN_INT (3), tmp, cmp));

  emit_label (end_0_label);
}

/* Expand strlen.  */

bool
ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
{
  rtx addr, scratch1, scratch2, scratch3, scratch4;

  /* The generic case of strlen expander is long.  Avoid it's
     expanding unless TARGET_INLINE_ALL_STRINGOPS.  */

  if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
      && !TARGET_INLINE_ALL_STRINGOPS
      && !optimize_insn_for_size_p ()
      && (!CONST_INT_P (align) || INTVAL (align) < 4))
    return false;

  addr = force_reg (Pmode, XEXP (src, 0));
  scratch1 = gen_reg_rtx (Pmode);

  if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
      && !optimize_insn_for_size_p ())
    {
      /* Well it seems that some optimizer does not combine a call like
         foo(strlen(bar), strlen(bar));
         when the move and the subtraction is done here.  It does calculate
         the length just once when these instructions are done inside of
         output_strlen_unroll().  But I think since &bar[strlen(bar)] is
         often used and I use one fewer register for the lifetime of
         output_strlen_unroll() this is better.  */

      emit_move_insn (out, addr);

      ix86_expand_strlensi_unroll_1 (out, src, align);

      /* strlensi_unroll_1 returns the address of the zero at the end of
         the string, like memchr(), so compute the length by subtracting
         the start address.  */
      emit_insn (ix86_gen_sub3 (out, out, addr));
    }
  else
    {
      rtx unspec;

      /* Can't use this if the user has appropriated eax, ecx, or edi.  */
      if (fixed_regs[AX_REG] || fixed_regs[CX_REG] || fixed_regs[DI_REG])
        return false;
      /* Can't use this for non-default address spaces.  */
      if (!ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (src)))
	return false;

      scratch2 = gen_reg_rtx (Pmode);
      scratch3 = gen_reg_rtx (Pmode);
      scratch4 = force_reg (Pmode, constm1_rtx);

      emit_move_insn (scratch3, addr);
      eoschar = force_reg (QImode, eoschar);

      src = replace_equiv_address_nv (src, scratch3);

      /* If .md starts supporting :P, this can be done in .md.  */
      unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, src, eoschar, align,
						 scratch4), UNSPEC_SCAS);
      emit_insn (gen_strlenqi_1 (scratch1, scratch3, unspec));
      emit_insn (ix86_gen_one_cmpl2 (scratch2, scratch1));
      emit_insn (ix86_gen_add3 (out, scratch2, constm1_rtx));
    }
  return true;
}

/* For given symbol (function) construct code to compute address of it's PLT
   entry in large x86-64 PIC model.  */
static rtx
construct_plt_address (rtx symbol)
{
  rtx tmp, unspec;

  gcc_assert (GET_CODE (symbol) == SYMBOL_REF);
  gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
  gcc_assert (Pmode == DImode);

  tmp = gen_reg_rtx (Pmode);
  unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);

  emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
  emit_insn (ix86_gen_add3 (tmp, tmp, pic_offset_table_rtx));
  return tmp;
}

rtx
ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
		  rtx callarg2,
		  rtx pop, bool sibcall)
{
  rtx vec[3];
  rtx use = NULL, call;
  unsigned int vec_len = 0;
  tree fndecl;

  if (GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
    {
      fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
      if (fndecl
	  && (lookup_attribute ("interrupt",
				TYPE_ATTRIBUTES (TREE_TYPE (fndecl)))))
	error ("interrupt service routine can't be called directly");
    }
  else
    fndecl = NULL_TREE;

  if (pop == const0_rtx)
    pop = NULL;
  gcc_assert (!TARGET_64BIT || !pop);

  if (TARGET_MACHO && !TARGET_64BIT)
    {
#if TARGET_MACHO
      if (flag_pic && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
	fnaddr = machopic_indirect_call_target (fnaddr);
#endif
    }
  else
    {
      /* Static functions and indirect calls don't need the pic register.  Also,
	 check if PLT was explicitly avoided via no-plt or "noplt" attribute, making
	 it an indirect call.  */
      rtx addr = XEXP (fnaddr, 0);
      if (flag_pic
	  && GET_CODE (addr) == SYMBOL_REF
	  && !SYMBOL_REF_LOCAL_P (addr))
	{
	  if (flag_plt
	      && (SYMBOL_REF_DECL (addr) == NULL_TREE
		  || !lookup_attribute ("noplt",
					DECL_ATTRIBUTES (SYMBOL_REF_DECL (addr)))))
	    {
	      if (!TARGET_64BIT
		  || (ix86_cmodel == CM_LARGE_PIC
		      && DEFAULT_ABI != MS_ABI))
		{
		  use_reg (&use, gen_rtx_REG (Pmode,
					      REAL_PIC_OFFSET_TABLE_REGNUM));
		  if (ix86_use_pseudo_pic_reg ())
		    emit_move_insn (gen_rtx_REG (Pmode,
						 REAL_PIC_OFFSET_TABLE_REGNUM),
				    pic_offset_table_rtx);
		}
	    }
	  else if (!TARGET_PECOFF && !TARGET_MACHO)
	    {
	      if (TARGET_64BIT)
		{
		  fnaddr = gen_rtx_UNSPEC (Pmode,
					   gen_rtvec (1, addr),
					   UNSPEC_GOTPCREL);
		  fnaddr = gen_rtx_CONST (Pmode, fnaddr);
		}
	      else
		{
		  fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
					   UNSPEC_GOT);
		  fnaddr = gen_rtx_CONST (Pmode, fnaddr);
		  fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
					 fnaddr);
		}
	      fnaddr = gen_const_mem (Pmode, fnaddr);
	      /* Pmode may not be the same as word_mode for x32, which
		 doesn't support indirect branch via 32-bit memory slot.
		 Since x32 GOT slot is 64 bit with zero upper 32 bits,
		 indirect branch via x32 GOT slot is OK.  */
	      if (GET_MODE (fnaddr) != word_mode)
		fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
	      fnaddr = gen_rtx_MEM (QImode, fnaddr);
	    }
	}
    }

  /* Skip setting up RAX register for -mskip-rax-setup when there are no
     parameters passed in vector registers.  */
  if (TARGET_64BIT
      && (INTVAL (callarg2) > 0
	  || (INTVAL (callarg2) == 0
	      && (TARGET_SSE || !flag_skip_rax_setup))))
    {
      rtx al = gen_rtx_REG (QImode, AX_REG);
      emit_move_insn (al, callarg2);
      use_reg (&use, al);
    }

  if (ix86_cmodel == CM_LARGE_PIC
      && !TARGET_PECOFF
      && MEM_P (fnaddr)
      && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
      && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
    fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
  /* Since x32 GOT slot is 64 bit with zero upper 32 bits, indirect
     branch via x32 GOT slot is OK.  */
  else if (!(TARGET_X32
	     && MEM_P (fnaddr)
	     && GET_CODE (XEXP (fnaddr, 0)) == ZERO_EXTEND
	     && GOT_memory_operand (XEXP (XEXP (fnaddr, 0), 0), Pmode))
	   && (sibcall
	       ? !sibcall_insn_operand (XEXP (fnaddr, 0), word_mode)
	       : !call_insn_operand (XEXP (fnaddr, 0), word_mode)))
    {
      fnaddr = convert_to_mode (word_mode, XEXP (fnaddr, 0), 1);
      fnaddr = gen_rtx_MEM (QImode, copy_to_mode_reg (word_mode, fnaddr));
    }

  call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);

  if (retval)
    {
      /* We should add bounds as destination register in case
	 pointer with bounds may be returned.  */
      if (TARGET_MPX && SCALAR_INT_MODE_P (GET_MODE (retval)))
	{
	  rtx b0 = gen_rtx_REG (BND64mode, FIRST_BND_REG);
	  rtx b1 = gen_rtx_REG (BND64mode, FIRST_BND_REG + 1);
	  if (GET_CODE (retval) == PARALLEL)
	    {
	      b0 = gen_rtx_EXPR_LIST (VOIDmode, b0, const0_rtx);
	      b1 = gen_rtx_EXPR_LIST (VOIDmode, b1, const0_rtx);
	      rtx par = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, b0, b1));
	      retval = chkp_join_splitted_slot (retval, par);
	    }
	  else
	    {
	      retval = gen_rtx_PARALLEL (VOIDmode,
					 gen_rtvec (3, retval, b0, b1));
	      chkp_put_regs_to_expr_list (retval);
	    }
	}

      call = gen_rtx_SET (retval, call);
    }
  vec[vec_len++] = call;

  if (pop)
    {
      pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
      pop = gen_rtx_SET (stack_pointer_rtx, pop);
      vec[vec_len++] = pop;
    }

  if (cfun->machine->no_caller_saved_registers
      && (!fndecl
	  || (!TREE_THIS_VOLATILE (fndecl)
	      && !lookup_attribute ("no_caller_saved_registers",
				    TYPE_ATTRIBUTES (TREE_TYPE (fndecl))))))
    {
      static const char ix86_call_used_regs[] = CALL_USED_REGISTERS;
      bool is_64bit_ms_abi = (TARGET_64BIT
			      && ix86_function_abi (fndecl) == MS_ABI);
      char c_mask = CALL_USED_REGISTERS_MASK (is_64bit_ms_abi);

      /* If there are no caller-saved registers, add all registers
	 that are clobbered by the call which returns.  */
      for (int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
	if (!fixed_regs[i]
	    && (ix86_call_used_regs[i] == 1
		|| (ix86_call_used_regs[i] & c_mask))
	    && !STACK_REGNO_P (i)
	    && !MMX_REGNO_P (i))
	  clobber_reg (&use,
		       gen_rtx_REG (GET_MODE (regno_reg_rtx[i]), i));
    }
  else if (TARGET_64BIT_MS_ABI
	   && (!callarg2 || INTVAL (callarg2) != -2))
    {
      int const cregs_size
	= ARRAY_SIZE (x86_64_ms_sysv_extra_clobbered_registers);
      int i;

      for (i = 0; i < cregs_size; i++)
	{
	  int regno = x86_64_ms_sysv_extra_clobbered_registers[i];
	  machine_mode mode = SSE_REGNO_P (regno) ? TImode : DImode;

	  clobber_reg (&use, gen_rtx_REG (mode, regno));
	}
    }

  if (vec_len > 1)
    call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (vec_len, vec));
  call = emit_call_insn (call);
  if (use)
    CALL_INSN_FUNCTION_USAGE (call) = use;

  return call;
}

/* Return true if the function being called was marked with attribute
   "noplt" or using -fno-plt and we are compiling for non-PIC.  We need
   to handle the non-PIC case in the backend because there is no easy
   interface for the front-end to force non-PLT calls to use the GOT.
   This is currently used only with 64-bit or 32-bit GOT32X ELF targets
   to call the function marked "noplt" indirectly.  */

static bool
ix86_nopic_noplt_attribute_p (rtx call_op)
{
  if (flag_pic || ix86_cmodel == CM_LARGE
      || !(TARGET_64BIT || HAVE_AS_IX86_GOT32X)
      || TARGET_MACHO || TARGET_SEH || TARGET_PECOFF
      || SYMBOL_REF_LOCAL_P (call_op))
    return false;

  tree symbol_decl = SYMBOL_REF_DECL (call_op);

  if (!flag_plt
      || (symbol_decl != NULL_TREE
          && lookup_attribute ("noplt", DECL_ATTRIBUTES (symbol_decl))))
    return true;

  return false;
}

/* Output the assembly for a call instruction.  */

const char *
ix86_output_call_insn (rtx_insn *insn, rtx call_op)
{
  bool direct_p = constant_call_address_operand (call_op, VOIDmode);
  bool seh_nop_p = false;
  const char *xasm;

  if (SIBLING_CALL_P (insn))
    {
      if (direct_p)
	{
	  if (ix86_nopic_noplt_attribute_p (call_op))
	    {
	      if (TARGET_64BIT)
		xasm = "%!jmp\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
	      else
		xasm = "%!jmp\t{*%p0@GOT|[DWORD PTR %p0@GOT]}";
	    }
	  else
	    xasm = "%!jmp\t%P0";
	}
      /* SEH epilogue detection requires the indirect branch case
	 to include REX.W.  */
      else if (TARGET_SEH)
	xasm = "%!rex.W jmp\t%A0";
      else
	xasm = "%!jmp\t%A0";

      output_asm_insn (xasm, &call_op);
      return "";
    }

  /* SEH unwinding can require an extra nop to be emitted in several
     circumstances.  Determine if we have one of those.  */
  if (TARGET_SEH)
    {
      rtx_insn *i;

      for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i))
	{
	  /* If we get to another real insn, we don't need the nop.  */
	  if (INSN_P (i))
	    break;

	  /* If we get to the epilogue note, prevent a catch region from
	     being adjacent to the standard epilogue sequence.  If non-
	     call-exceptions, we'll have done this during epilogue emission. */
	  if (NOTE_P (i) && NOTE_KIND (i) == NOTE_INSN_EPILOGUE_BEG
	      && !flag_non_call_exceptions
	      && !can_throw_internal (insn))
	    {
	      seh_nop_p = true;
	      break;
	    }
	}

      /* If we didn't find a real insn following the call, prevent the
	 unwinder from looking into the next function.  */
      if (i == NULL)
	seh_nop_p = true;
    }

  if (direct_p)
    {
      if (ix86_nopic_noplt_attribute_p (call_op))
	{
	  if (TARGET_64BIT)
	    xasm = "%!call\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
	  else
	    xasm = "%!call\t{*%p0@GOT|[DWORD PTR %p0@GOT]}";
	}
      else
	xasm = "%!call\t%P0";
    }
  else
    xasm = "%!call\t%A0";

  output_asm_insn (xasm, &call_op);

  if (seh_nop_p)
    return "nop";

  return "";
}

/* Clear stack slot assignments remembered from previous functions.
   This is called from INIT_EXPANDERS once before RTL is emitted for each
   function.  */

static struct machine_function *
ix86_init_machine_status (void)
{
  struct machine_function *f;

  f = ggc_cleared_alloc<machine_function> ();
  f->use_fast_prologue_epilogue_nregs = -1;
  f->call_abi = ix86_abi;

  return f;
}

/* Return a MEM corresponding to a stack slot with mode MODE.
   Allocate a new slot if necessary.

   The RTL for a function can have several slots available: N is
   which slot to use.  */

rtx
assign_386_stack_local (machine_mode mode, enum ix86_stack_slot n)
{
  struct stack_local_entry *s;

  gcc_assert (n < MAX_386_STACK_LOCALS);

  for (s = ix86_stack_locals; s; s = s->next)
    if (s->mode == mode && s->n == n)
      return validize_mem (copy_rtx (s->rtl));

  s = ggc_alloc<stack_local_entry> ();
  s->n = n;
  s->mode = mode;
  s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), 0);

  s->next = ix86_stack_locals;
  ix86_stack_locals = s;
  return validize_mem (copy_rtx (s->rtl));
}

static void
ix86_instantiate_decls (void)
{
  struct stack_local_entry *s;

  for (s = ix86_stack_locals; s; s = s->next)
    if (s->rtl != NULL_RTX)
      instantiate_decl_rtl (s->rtl);
}

/* Return the number used for encoding REG, in the range 0..7.  */

static int
reg_encoded_number (rtx reg)
{
  unsigned regno = REGNO (reg);
  switch (regno)
    {
    case AX_REG:
      return 0;
    case CX_REG:
      return 1;
    case DX_REG:
      return 2;
    case BX_REG:
      return 3;
    case SP_REG:
      return 4;
    case BP_REG:
      return 5;
    case SI_REG:
      return 6;
    case DI_REG:
      return 7;
    default:
      break;
    }
  if (IN_RANGE (regno, FIRST_STACK_REG, LAST_STACK_REG))
    return regno - FIRST_STACK_REG;
  if (IN_RANGE (regno, FIRST_SSE_REG, LAST_SSE_REG))
    return regno - FIRST_SSE_REG;
  if (IN_RANGE (regno, FIRST_MMX_REG, LAST_MMX_REG))
    return regno - FIRST_MMX_REG;
  if (IN_RANGE (regno, FIRST_REX_SSE_REG, LAST_REX_SSE_REG))
    return regno - FIRST_REX_SSE_REG;
  if (IN_RANGE (regno, FIRST_REX_INT_REG, LAST_REX_INT_REG))
    return regno - FIRST_REX_INT_REG;
  if (IN_RANGE (regno, FIRST_MASK_REG, LAST_MASK_REG))
    return regno - FIRST_MASK_REG;
  if (IN_RANGE (regno, FIRST_BND_REG, LAST_BND_REG))
    return regno - FIRST_BND_REG;
  return -1;
}

/* Given an insn INSN with NOPERANDS OPERANDS, return the modr/m byte used
   in its encoding if it could be relevant for ROP mitigation, otherwise
   return -1.  If POPNO0 and POPNO1 are nonnull, store the operand numbers
   used for calculating it into them.  */

static int
ix86_get_modrm_for_rop (rtx_insn *insn, rtx *operands, int noperands,
			int *popno0 = 0, int *popno1 = 0)
{
  if (asm_noperands (PATTERN (insn)) >= 0)
    return -1;
  int has_modrm = get_attr_modrm (insn);
  if (!has_modrm)
    return -1;
  enum attr_modrm_class cls = get_attr_modrm_class (insn);
  rtx op0, op1;
  switch (cls)
    {
    case MODRM_CLASS_OP02:
      gcc_assert (noperands >= 3);
      if (popno0)
	{
	  *popno0 = 0;
	  *popno1 = 2;
	}
      op0 = operands[0];
      op1 = operands[2];
      break;
    case MODRM_CLASS_OP01:
      gcc_assert (noperands >= 2);
      if (popno0)
	{
	  *popno0 = 0;
	  *popno1 = 1;
	}
      op0 = operands[0];
      op1 = operands[1];
      break;
    default:
      return -1;
    }
  if (REG_P (op0) && REG_P (op1))
    {
      int enc0 = reg_encoded_number (op0);
      int enc1 = reg_encoded_number (op1);
      return 0xc0 + (enc1 << 3) + enc0;
    }
  return -1;
}

/* Check whether x86 address PARTS is a pc-relative address.  */

static bool
rip_relative_addr_p (struct ix86_address *parts)
{
  rtx base, index, disp;

  base = parts->base;
  index = parts->index;
  disp = parts->disp;

  if (disp && !base && !index)
    {
      if (TARGET_64BIT)
	{
	  rtx symbol = disp;

	  if (GET_CODE (disp) == CONST)
	    symbol = XEXP (disp, 0);
	  if (GET_CODE (symbol) == PLUS
	      && CONST_INT_P (XEXP (symbol, 1)))
	    symbol = XEXP (symbol, 0);

	  if (GET_CODE (symbol) == LABEL_REF
	      || (GET_CODE (symbol) == SYMBOL_REF
		  && SYMBOL_REF_TLS_MODEL (symbol) == 0)
	      || (GET_CODE (symbol) == UNSPEC
		  && (XINT (symbol, 1) == UNSPEC_GOTPCREL
		      || XINT (symbol, 1) == UNSPEC_PCREL
		      || XINT (symbol, 1) == UNSPEC_GOTNTPOFF)))
	    return true;
	}
    }
  return false;
}

/* Calculate the length of the memory address in the instruction encoding.
   Includes addr32 prefix, does not include the one-byte modrm, opcode,
   or other prefixes.  We never generate addr32 prefix for LEA insn.  */

int
memory_address_length (rtx addr, bool lea)
{
  struct ix86_address parts;
  rtx base, index, disp;
  int len;
  int ok;

  if (GET_CODE (addr) == PRE_DEC
      || GET_CODE (addr) == POST_INC
      || GET_CODE (addr) == PRE_MODIFY
      || GET_CODE (addr) == POST_MODIFY)
    return 0;

  ok = ix86_decompose_address (addr, &parts);
  gcc_assert (ok);

  len = (parts.seg == ADDR_SPACE_GENERIC) ? 0 : 1;

  /*  If this is not LEA instruction, add the length of addr32 prefix.  */
  if (TARGET_64BIT && !lea
      && (SImode_address_operand (addr, VOIDmode)
	  || (parts.base && GET_MODE (parts.base) == SImode)
	  || (parts.index && GET_MODE (parts.index) == SImode)))
    len++;

  base = parts.base;
  index = parts.index;
  disp = parts.disp;

  if (base && SUBREG_P (base))
    base = SUBREG_REG (base);
  if (index && SUBREG_P (index))
    index = SUBREG_REG (index);

  gcc_assert (base == NULL_RTX || REG_P (base));
  gcc_assert (index == NULL_RTX || REG_P (index));

  /* Rule of thumb:
       - esp as the base always wants an index,
       - ebp as the base always wants a displacement,
       - r12 as the base always wants an index,
       - r13 as the base always wants a displacement.  */

  /* Register Indirect.  */
  if (base && !index && !disp)
    {
      /* esp (for its index) and ebp (for its displacement) need
	 the two-byte modrm form.  Similarly for r12 and r13 in 64-bit
	 code.  */
      if (base == arg_pointer_rtx
	  || base == frame_pointer_rtx
	  || REGNO (base) == SP_REG
	  || REGNO (base) == BP_REG
	  || REGNO (base) == R12_REG
	  || REGNO (base) == R13_REG)
	len++;
    }

  /* Direct Addressing.  In 64-bit mode mod 00 r/m 5
     is not disp32, but disp32(%rip), so for disp32
     SIB byte is needed, unless print_operand_address
     optimizes it into disp32(%rip) or (%rip) is implied
     by UNSPEC.  */
  else if (disp && !base && !index)
    {
      len += 4;
      if (!rip_relative_addr_p (&parts))
	len++;
    }
  else
    {
      /* Find the length of the displacement constant.  */
      if (disp)
	{
	  if (base && satisfies_constraint_K (disp))
	    len += 1;
	  else
	    len += 4;
	}
      /* ebp always wants a displacement.  Similarly r13.  */
      else if (base && (REGNO (base) == BP_REG || REGNO (base) == R13_REG))
	len++;

      /* An index requires the two-byte modrm form....  */
      if (index
	  /* ...like esp (or r12), which always wants an index.  */
	  || base == arg_pointer_rtx
	  || base == frame_pointer_rtx
	  || (base && (REGNO (base) == SP_REG || REGNO (base) == R12_REG)))
	len++;
    }

  return len;
}

/* Compute default value for "length_immediate" attribute.  When SHORTFORM
   is set, expect that insn have 8bit immediate alternative.  */
int
ix86_attr_length_immediate_default (rtx_insn *insn, bool shortform)
{
  int len = 0;
  int i;
  extract_insn_cached (insn);
  for (i = recog_data.n_operands - 1; i >= 0; --i)
    if (CONSTANT_P (recog_data.operand[i]))
      {
        enum attr_mode mode = get_attr_mode (insn);

	gcc_assert (!len);
	if (shortform && CONST_INT_P (recog_data.operand[i]))
	  {
	    HOST_WIDE_INT ival = INTVAL (recog_data.operand[i]);
	    switch (mode)
	      {
	      case MODE_QI:
		len = 1;
		continue;
	      case MODE_HI:
		ival = trunc_int_for_mode (ival, HImode);
		break;
	      case MODE_SI:
		ival = trunc_int_for_mode (ival, SImode);
		break;
	      default:
		break;
	      }
	    if (IN_RANGE (ival, -128, 127))
	      {
		len = 1;
		continue;
	      }
	  }
	switch (mode)
	  {
	  case MODE_QI:
	    len = 1;
	    break;
	  case MODE_HI:
	    len = 2;
	    break;
	  case MODE_SI:
	    len = 4;
	    break;
	  /* Immediates for DImode instructions are encoded
	     as 32bit sign extended values.  */
	  case MODE_DI:
	    len = 4;
	    break;
	  default:
	    fatal_insn ("unknown insn mode", insn);
	}
      }
  return len;
}

/* Compute default value for "length_address" attribute.  */
int
ix86_attr_length_address_default (rtx_insn *insn)
{
  int i;

  if (get_attr_type (insn) == TYPE_LEA)
    {
      rtx set = PATTERN (insn), addr;

      if (GET_CODE (set) == PARALLEL)
	set = XVECEXP (set, 0, 0);

      gcc_assert (GET_CODE (set) == SET);

      addr = SET_SRC (set);

      return memory_address_length (addr, true);
    }

  extract_insn_cached (insn);
  for (i = recog_data.n_operands - 1; i >= 0; --i)
    {
      rtx op = recog_data.operand[i];
      if (MEM_P (op))
	{
	  constrain_operands_cached (insn, reload_completed);
	  if (which_alternative != -1)
	    {
	      const char *constraints = recog_data.constraints[i];
	      int alt = which_alternative;

	      while (*constraints == '=' || *constraints == '+')
		constraints++;
	      while (alt-- > 0)