Fold single imm use of a FMA if it is a negation [PR95060]

match.pd already has simplifications for negation of a FMA (FMS, FNMA, FNMS)
call if it is single use, but when the widening_mul pass discovers FMAs,
nothing folds the statements anymore.

So, the following patch adjusts the widening_mul pass to handle that.

I had to adjust quite a lot of tests, because they have in them nested FMAs
(one FMA feeding another one) and the patch results in some (equivalent) changes
in the chosen instructions, previously the negation of one FMA's result
would result in the dependent FMA being adjusted for the negation, but now
instead the first FMA is adjusted.

2020-05-13  Jakub Jelinek  <jakub@redhat.com>

	PR tree-optimization/95060
	* tree-ssa-math-opts.c (convert_mult_to_fma_1): Fold a NEGATE_EXPR
	if it is the single use of the FMA internal builtin.

	* gcc.target/i386/avx512f-pr95060.c: New test.
	* gcc.target/i386/fma_double_1.c: Adjust expected insn counts.
	* gcc.target/i386/fma_double_2.c: Likewise.
	* gcc.target/i386/fma_double_3.c: Likewise.
	* gcc.target/i386/fma_double_4.c: Likewise.
	* gcc.target/i386/fma_double_5.c: Likewise.
	* gcc.target/i386/fma_double_6.c: Likewise.
	* gcc.target/i386/fma_float_1.c: Likewise.
	* gcc.target/i386/fma_float_2.c: Likewise.
	* gcc.target/i386/fma_float_3.c: Likewise.
	* gcc.target/i386/fma_float_4.c: Likewise.
	* gcc.target/i386/fma_float_5.c: Likewise.
	* gcc.target/i386/fma_float_6.c: Likewise.
	* gcc.target/i386/l_fma_double_1.c: Likewise.
	* gcc.target/i386/l_fma_double_2.c: Likewise.
	* gcc.target/i386/l_fma_double_3.c: Likewise.
	* gcc.target/i386/l_fma_double_4.c: Likewise.
	* gcc.target/i386/l_fma_double_5.c: Likewise.
	* gcc.target/i386/l_fma_double_6.c: Likewise.
	* gcc.target/i386/l_fma_float_1.c: Likewise.
	* gcc.target/i386/l_fma_float_2.c: Likewise.
	* gcc.target/i386/l_fma_float_3.c: Likewise.
	* gcc.target/i386/l_fma_float_4.c: Likewise.
	* gcc.target/i386/l_fma_float_5.c: Likewise.
	* gcc.target/i386/l_fma_float_6.c: Likewise.

28 files changed, 226 insertions, 144 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 4234a72..ba230b4 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,9 @@
+2020-05-13  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/95060
+	* tree-ssa-math-opts.c (convert_mult_to_fma_1): Fold a NEGATE_EXPR
+	if it is the single use of the FMA internal builtin.
+
 2020-05-13  Bin Cheng  <bin.cheng@linux.alibaba.com>
 
 	PR tree-optimization/94969
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 3e5fe75..02878c0 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,32 @@
+2020-05-13  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/95060
+	* gcc.target/i386/avx512f-pr95060.c: New test.
+	* gcc.target/i386/fma_double_1.c: Adjust expected insn counts.
+	* gcc.target/i386/fma_double_2.c: Likewise.
+	* gcc.target/i386/fma_double_3.c: Likewise.
+	* gcc.target/i386/fma_double_4.c: Likewise.
+	* gcc.target/i386/fma_double_5.c: Likewise.
+	* gcc.target/i386/fma_double_6.c: Likewise.
+	* gcc.target/i386/fma_float_1.c: Likewise.
+	* gcc.target/i386/fma_float_2.c: Likewise.
+	* gcc.target/i386/fma_float_3.c: Likewise.
+	* gcc.target/i386/fma_float_4.c: Likewise.
+	* gcc.target/i386/fma_float_5.c: Likewise.
+	* gcc.target/i386/fma_float_6.c: Likewise.
+	* gcc.target/i386/l_fma_double_1.c: Likewise.
+	* gcc.target/i386/l_fma_double_2.c: Likewise.
+	* gcc.target/i386/l_fma_double_3.c: Likewise.
+	* gcc.target/i386/l_fma_double_4.c: Likewise.
+	* gcc.target/i386/l_fma_double_5.c: Likewise.
+	* gcc.target/i386/l_fma_double_6.c: Likewise.
+	* gcc.target/i386/l_fma_float_1.c: Likewise.
+	* gcc.target/i386/l_fma_float_2.c: Likewise.
+	* gcc.target/i386/l_fma_float_3.c: Likewise.
+	* gcc.target/i386/l_fma_float_4.c: Likewise.
+	* gcc.target/i386/l_fma_float_5.c: Likewise.
+	* gcc.target/i386/l_fma_float_6.c: Likewise.
+
 2020-05-13  Martin Liska  <mliska@suse.cz>
 
 	PR sanitizer/95051
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-pr95060.c b/gcc/testsuite/gcc.target/i386/avx512f-pr95060.c
new file mode 100644
index 0000000..b38dc4f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512f-pr95060.c
@@ -0,0 +1,22 @@
+/* PR tree-optimization/95060 */
+/* { dg-do compile } */
+/* { dg-options "-O3 -ffast-math -mavx512f" } */
+/* { dg-final { scan-assembler "\tvfnmsub" } } */
+/* { dg-final { scan-assembler-not "\tvfmadd" } } */
+
+#define N 32
+float r[N], a[N], b[N], c[N];
+
+void
+foo (void)
+{
+  for (int i = 0; i < N; i++)
+    r[i] = -(a[i] * b[i]) - c[i];
+}
+
+void
+bar (void)
+{
+  for (int i = 0; i < N; i++)
+    r[i] = -(a[i] * b[i] + c[i]);
+}
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_1.c b/gcc/testsuite/gcc.target/i386/fma_double_1.c
index c3aa3e8..767ee5c 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_1.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_1.c
@@ -8,11 +8,9 @@
 
 #include "fma_1.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfmadd132sd" 8  } } */
 /* { dg-final { scan-assembler-times "vfmadd231sd" 4  } } */
-/* { dg-final { scan-assembler-times "vfmsub132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfmsub132sd" 8  } } */
 /* { dg-final { scan-assembler-times "vfmsub231sd" 4  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132sd" 4  } } */
 /* { dg-final { scan-assembler-times "vfnmadd231sd" 4  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132sd" 4  } } */
 /* { dg-final { scan-assembler-times "vfnmsub231sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_2.c b/gcc/testsuite/gcc.target/i386/fma_double_2.c
index 843eff0..f15fb3b 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_2.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_2.c
@@ -8,7 +8,7 @@
 
 #include "fma_2.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_3.c b/gcc/testsuite/gcc.target/i386/fma_double_3.c
index 3a04777..6b67774 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_3.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_3.c
@@ -8,7 +8,7 @@
 
 #include "fma_3.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[132\]+sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub\[132\]+sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[132\]+sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[132\]+sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_4.c b/gcc/testsuite/gcc.target/i386/fma_double_4.c
index 51fc111..267f6fd 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_4.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_4.c
@@ -8,7 +8,7 @@
 
 #include "fma_4.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_5.c b/gcc/testsuite/gcc.target/i386/fma_double_5.c
index 640b552..fd7fcab 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_5.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_5.c
@@ -8,7 +8,7 @@
 
 #include "fma_5.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[132\]+sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd\[132\]+sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub\[132\]+sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[132\]+sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[132\]+sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_6.c b/gcc/testsuite/gcc.target/i386/fma_double_6.c
index 7b75a22..f7ab1ce 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_6.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_6.c
@@ -8,7 +8,7 @@
 
 #include "fma_6.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_1.c b/gcc/testsuite/gcc.target/i386/fma_float_1.c
index 67b1f3f..a4ac2cb 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_1.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_1.c
@@ -8,11 +8,9 @@
 
 #include "fma_1.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfmadd132ss" 8  } } */
 /* { dg-final { scan-assembler-times "vfmadd231ss" 4  } } */
-/* { dg-final { scan-assembler-times "vfmsub132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfmsub132ss" 8  } } */
 /* { dg-final { scan-assembler-times "vfmsub231ss" 4  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132ss" 4  } } */
 /* { dg-final { scan-assembler-times "vfnmadd231ss" 4  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132ss" 4  } } */
 /* { dg-final { scan-assembler-times "vfnmsub231ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_2.c b/gcc/testsuite/gcc.target/i386/fma_float_2.c
index a54644d..a750911 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_2.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_2.c
@@ -8,7 +8,7 @@
 
 #include "fma_2.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_3.c b/gcc/testsuite/gcc.target/i386/fma_float_3.c
index 7986ce4..d88a3bf 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_3.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_3.c
@@ -8,7 +8,7 @@
 
 #include "fma_3.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[132\]+ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd\[132\]+ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub\[132\]+ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[132\]+ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[132\]+ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_4.c b/gcc/testsuite/gcc.target/i386/fma_float_4.c
index d9689d9..cb1a81c 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_4.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_4.c
@@ -8,7 +8,7 @@
 
 #include "fma_4.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_5.c b/gcc/testsuite/gcc.target/i386/fma_float_5.c
index 2105ae6..3a62590 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_5.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_5.c
@@ -8,7 +8,7 @@
 
 #include "fma_5.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[132\]+ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd\[132\]+ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub\[132\]+ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[132\]+ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[132\]+ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_6.c b/gcc/testsuite/gcc.target/i386/fma_float_6.c
index c758073..30d8283 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_6.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_6.c
@@ -8,7 +8,7 @@
 
 #include "fma_6.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_1.c b/gcc/testsuite/gcc.target/i386/l_fma_double_1.c
index aa7c130..5089874 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_1.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_1.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_1.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_2.c b/gcc/testsuite/gcc.target/i386/l_fma_double_2.c
index c59a891..e469620 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_2.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_2.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_2.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_3.c b/gcc/testsuite/gcc.target/i386/l_fma_double_3.c
index c508d64..df986d0 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_3.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_3.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_3.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_4.c b/gcc/testsuite/gcc.target/i386/l_fma_double_4.c
index fec33e4..ae06559 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_4.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_4.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_4.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_5.c b/gcc/testsuite/gcc.target/i386/l_fma_double_5.c
index 8bcfe4e..5d31aba 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_5.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_5.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_5.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_6.c b/gcc/testsuite/gcc.target/i386/l_fma_double_6.c
index 15da66b..ff857fb 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_6.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_6.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_6.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_1.c b/gcc/testsuite/gcc.target/i386/l_fma_float_1.c
index 29eb77f..daef876 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_1.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_1.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_1.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_2.c b/gcc/testsuite/gcc.target/i386/l_fma_float_2.c
index 2943b2c..ffa5c6f 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_2.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_2.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_2.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_3.c b/gcc/testsuite/gcc.target/i386/l_fma_float_3.c
index c144dc1..a05ef59 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_3.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_3.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_3.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_4.c b/gcc/testsuite/gcc.target/i386/l_fma_float_4.c
index a940dfd..b0a37ba 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_4.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_4.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_4.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_5.c b/gcc/testsuite/gcc.target/i386/l_fma_float_5.c
index e7a12d4..598275c 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_5.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_5.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_5.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_6.c b/gcc/testsuite/gcc.target/i386/l_fma_float_6.c
index 82397a9..1e7a216 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_6.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_6.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_6.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/tree-ssa-math-opts.c b/gcc/tree-ssa-math-opts.c
index 969c1a6..5fbaa24 100644
--- a/gcc/tree-ssa-math-opts.c
+++ b/gcc/tree-ssa-math-opts.c
@@ -2930,6 +2930,35 @@ convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2)
 	  fprintf (dump_file, "\n");
 	}
 
+      /* If the FMA result is negated in a single use, fold the negation
+	 too.  */
+      orig_stmt = gsi_stmt (gsi);
+      use_operand_p use_p;
+      gimple *neg_stmt;
+      if (is_gimple_call (orig_stmt)
+	  && gimple_call_internal_p (orig_stmt)
+	  && gimple_call_lhs (orig_stmt)
+	  && TREE_CODE (gimple_call_lhs (orig_stmt)) == SSA_NAME
+	  && single_imm_use (gimple_call_lhs (orig_stmt), &use_p, &neg_stmt)
+	  && is_gimple_assign (neg_stmt)
+	  && gimple_assign_rhs_code (neg_stmt) == NEGATE_EXPR
+	  && !stmt_could_throw_p (cfun, neg_stmt))
+	{
+	  gsi = gsi_for_stmt (neg_stmt);
+	  if (fold_stmt (&gsi, follow_all_ssa_edges))
+	    {
+	      if (maybe_clean_or_replace_eh_stmt (neg_stmt, gsi_stmt (gsi)))
+		gcc_unreachable ();
+	      update_stmt (gsi_stmt (gsi));
+	      if (dump_file && (dump_flags & TDF_DETAILS))
+		{
+		  fprintf (dump_file, "Folded FMA negation ");
+		  print_gimple_stmt (dump_file, gsi_stmt (gsi), 0, TDF_NONE);
+		  fprintf (dump_file, "\n");
+		}
+	    }
+	}
+
       widen_mul_stats.fmas_inserted++;
     }
 }