8 files changed, 41 insertions, 28 deletions
diff --git a/gcc/testsuite/g++.dg/tree-ssa/pr86544.C b/gcc/testsuite/g++.dg/tree-ssa/pr86544.C
index ef43891..50befb3 100644
--- a/gcc/testsuite/g++.dg/tree-ssa/pr86544.C
+++ b/gcc/testsuite/g++.dg/tree-ssa/pr86544.C
@@ -12,5 +12,5 @@ int PopCount (long b) {
     return c;
 }
 
-/* { dg-final { scan-tree-dump-times "__builtin_popcount" 1 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "__builtin_popcount|\\.POPCOUNT" 1 "optimized" } } */
 /* { dg-final { scan-tree-dump-times "if" 0 "phiopt4" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/popcount.c b/gcc/testsuite/gcc.dg/tree-ssa/popcount.c
index b469410..efd906a 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/popcount.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/popcount.c
@@ -39,4 +39,4 @@ void PopCount3 (long b1) {
       }
 }
 
-/* { dg-final { scan-tree-dump-times "__builtin_popcount" 3 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "__builtin_popcount|\\.POPCOUNT" 3 "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/popcount2.c b/gcc/testsuite/gcc.dg/tree-ssa/popcount2.c
index ef73e34..ae38a32 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/popcount2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/popcount2.c
@@ -26,4 +26,4 @@ int main()
   return 0;
 }
 
-/* { dg-final { scan-tree-dump-times "__builtin_popcount" 1 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "__builtin_popcount|\\.POPCOUNT" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/popcount3.c b/gcc/testsuite/gcc.dg/tree-ssa/popcount3.c
index ef43891..50befb3 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/popcount3.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/popcount3.c
@@ -12,5 +12,5 @@ int PopCount (long b) {
     return c;
 }
 
-/* { dg-final { scan-tree-dump-times "__builtin_popcount" 1 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "__builtin_popcount|\\.POPCOUNT" 1 "optimized" } } */
 /* { dg-final { scan-tree-dump-times "if" 0 "phiopt4" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/popcount4.c b/gcc/testsuite/gcc.target/aarch64/popcount4.c
index ee55b2e..8aa15ff 100644
--- a/gcc/testsuite/gcc.target/aarch64/popcount4.c
+++ b/gcc/testsuite/gcc.target/aarch64/popcount4.c
@@ -11,4 +11,4 @@ int PopCount (long b) {
     return c;
 }
 
-/* { dg-final { scan-tree-dump-times "__builtin_popcount" 0 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "__builtin_popcount|\\.POPCOUNT" 0 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr95771-2.c b/gcc/testsuite/gcc.target/i386/pr95771-2.c
new file mode 100644
index 0000000..1db9dc9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr95771-2.c
@@ -0,0 +1,17 @@
+/* PR tree-optimization/95771 */
+/* { dg-do compile } */
+/* { dg-require-effective-target int128 } */
+/* { dg-options "-O2 -mpopcnt -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump " = __builtin_popcount| = \\.POPCOUNT" "optimized" } } */
+
+int
+corge (unsigned __int128 x)
+{
+  int i = 0;
+  while (x)
+    {
+      x &= x - 1;
+      ++i;
+    }
+  return i;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr95771.c b/gcc/testsuite/gcc.target/i386/pr95771.c
index d7b67017..d41be44 100644
--- a/gcc/testsuite/gcc.target/i386/pr95771.c
+++ b/gcc/testsuite/gcc.target/i386/pr95771.c
@@ -1,8 +1,7 @@
 /* PR tree-optimization/95771 */
 /* { dg-do compile } */
 /* { dg-options "-O2 -mpopcnt -fdump-tree-optimized" } */
-/* { dg-final { scan-tree-dump-times " = __builtin_popcount" 6 "optimized" { target int128 } } } */
-/* { dg-final { scan-tree-dump-times " = __builtin_popcount" 4 "optimized" { target { ! int128 } } } } */
+/* { dg-final { scan-tree-dump-times " = __builtin_popcount| = \\.POPCOUNT" 4 "optimized" } } */
 
 int
 foo (unsigned char x)
@@ -51,17 +50,3 @@ qux (unsigned long long x)
     }
   return i;
 }
-
-#ifdef __SIZEOF_INT128__
-int
-corge (unsigned __int128 x)
-{
-  int i = 0;
-  while (x)
-    {
-      x &= x - 1;
-      ++i;
-    }
-  return i;
-}
-#endif
diff --git a/gcc/tree-ssa-loop-niter.cc b/gcc/tree-ssa-loop-niter.cc
index 26e39ad..65b9604 100644
--- a/gcc/tree-ssa-loop-niter.cc
+++ b/gcc/tree-ssa-loop-niter.cc
@@ -2033,11 +2033,18 @@ static tree
 build_popcount_expr (tree src)
 {
   tree fn;
+  bool use_ifn = false;
   int prec = TYPE_PRECISION (TREE_TYPE (src));
   int i_prec = TYPE_PRECISION (integer_type_node);
   int li_prec = TYPE_PRECISION (long_integer_type_node);
   int lli_prec = TYPE_PRECISION (long_long_integer_type_node);
-  if (prec <= i_prec)
+
+  tree utype = unsigned_type_for (TREE_TYPE (src));
+  src = fold_convert (utype, src);
+
+  if (direct_internal_fn_supported_p (IFN_POPCOUNT, utype, OPTIMIZE_FOR_BOTH))
+    use_ifn = true;
+  else if (prec <= i_prec)
     fn = builtin_decl_implicit (BUILT_IN_POPCOUNT);
   else if (prec == li_prec)
     fn = builtin_decl_implicit (BUILT_IN_POPCOUNTL);
@@ -2046,12 +2053,11 @@ build_popcount_expr (tree src)
   else
     return NULL_TREE;
 
-  tree utype = unsigned_type_for (TREE_TYPE (src));
-  src = fold_convert (utype, src);
-  if (prec < i_prec)
-    src = fold_convert (unsigned_type_node, src);
   tree call;
-  if (prec == 2 * lli_prec)
+  if (use_ifn)
+      call = build_call_expr_internal_loc (UNKNOWN_LOCATION, IFN_POPCOUNT,
+					   integer_type_node, 1, src);
+  else if (prec == 2 * lli_prec)
     {
       tree src1 = fold_convert (long_long_unsigned_type_node,
 				fold_build2 (RSHIFT_EXPR, TREE_TYPE (src),
@@ -2064,7 +2070,12 @@ build_popcount_expr (tree src)
       call = fold_build2 (PLUS_EXPR, integer_type_node, call1, call2);
     }
   else
-    call = build_call_expr (fn, 1, src);
+    {
+      if (prec < i_prec)
+	src = fold_convert (unsigned_type_node, src);
+
+      call = build_call_expr (fn, 1, src);
+    }
 
   return call;
 }