[AArch64] Use atomic load-operate instructions for update-fetch patterns.

2015-09-22  Matthew Wahab  <matthew.wahab@arm.com>

	* config/aarch64/aarch64-protos.h (aarch64_gen_atomic_ldop):
	Adjust declaration.
	* config/aarch64/aarch64.c (aarch64_emit_bic): New.
	(aarch64_gen_atomic_ldop): Adjust comment.  Add parameter
	out_result.  Update to support update-fetch operations.
	* config/aarch64/atomics.md (aarch64_atomic_exchange<mode>_lse):
	Adjust for change to aarch64_gen_atomic_ldop.
	(aarch64_atomic_<atomic_optab><mode>_lse): Likewise.
	(aarch64_atomic_fetch_<atomic_optab><mode>_lse): Likewise.
	(atomic_<atomic_optab>_fetch<mode>): Change to an expander.
	(aarch64_atomic_<atomic_optab>_fetch<mode>): New.
	(aarch64_atomic_<atomic_optab>_fetch<mode>_lse): New.

gcc/testsuite
2015-09-22  Matthew Wahab  <matthew.wahab@arm.com>

	* gcc.target/aarch64/atomic-inst-ldadd.c: Add tests for
	update-fetch operations.
	* gcc.target/aarch64/atomic-inst-ldlogic.c: Likewise.

From-SVN: r228002

3 files changed, 118 insertions, 11 deletions

diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index 76ebd6f..dd8ebcc 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -380,7 +380,7 @@ void aarch64_split_compare_and_swap (rtx op[]);
 void aarch64_gen_atomic_cas (rtx, rtx, rtx, rtx, rtx);
 
 bool aarch64_atomic_ldop_supported_p (enum rtx_code);
-void aarch64_gen_atomic_ldop (enum rtx_code, rtx, rtx, rtx, rtx);
+void aarch64_gen_atomic_ldop (enum rtx_code, rtx, rtx, rtx, rtx, rtx);
 void aarch64_split_atomic_op (enum rtx_code, rtx, rtx, rtx, rtx, rtx, rtx);
 
 bool aarch64_gen_adjusted_ldpstp (rtx *, bool, enum machine_mode, RTX_CODE);
diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index 93e36b7..3c8c058 100644
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -11018,6 +11018,25 @@ aarch64_split_compare_and_swap (rtx operands[])
     aarch64_emit_post_barrier (model);
 }
 
+/* Emit a BIC instruction.  */
+
+static void
+aarch64_emit_bic (machine_mode mode, rtx dst, rtx s1, rtx s2, int shift)
+{
+  rtx shift_rtx = GEN_INT (shift);
+  rtx (*gen) (rtx, rtx, rtx, rtx);
+
+  switch (mode)
+    {
+    case SImode: gen = gen_and_one_cmpl_lshrsi3; break;
+    case DImode: gen = gen_and_one_cmpl_lshrdi3; break;
+    default:
+      gcc_unreachable ();
+    }
+
+  emit_insn (gen (dst, s2, shift_rtx, s1));
+}
+
 /* Emit an atomic swap.  */
 
 static void
@@ -11112,13 +11131,14 @@ aarch64_emit_atomic_load_op (enum aarch64_atomic_load_op_code code,
 }
 
 /* Emit an atomic load+operate.  CODE is the operation.  OUT_DATA is the
-   location to store the data read from memory.  MEM is the memory location to
-   read and modify.  MODEL_RTX is the memory ordering to use.  VALUE is the
-   second operand for the operation.  Either OUT_DATA or OUT_RESULT, but not
-   both, can be NULL.  */
+   location to store the data read from memory.  OUT_RESULT is the location to
+   store the result of the operation.  MEM is the memory location to read and
+   modify.  MODEL_RTX is the memory ordering to use.  VALUE is the second
+   operand for the operation.  Either OUT_DATA or OUT_RESULT, but not both, can
+   be NULL.  */
 
 void
-aarch64_gen_atomic_ldop (enum rtx_code code, rtx out_data,
+aarch64_gen_atomic_ldop (enum rtx_code code, rtx out_data, rtx out_result,
 			 rtx mem, rtx value, rtx model_rtx)
 {
   machine_mode mode = GET_MODE (mem);
@@ -11131,12 +11151,15 @@ aarch64_gen_atomic_ldop (enum rtx_code code, rtx out_data,
   if (out_data)
     out_data = gen_lowpart (mode, out_data);
 
+  if (out_result)
+    out_result = gen_lowpart (mode, out_result);
+
   /* Make sure the value is in a register, putting it into a destination
      register if it needs to be manipulated.  */
   if (!register_operand (value, mode)
       || code == AND || code == MINUS)
     {
-      src = out_data;
+      src = out_result ? out_result : out_data;
       emit_move_insn (src, gen_lowpart (mode, value));
     }
   else
@@ -11202,6 +11225,43 @@ aarch64_gen_atomic_ldop (enum rtx_code code, rtx out_data,
     }
 
   aarch64_emit_atomic_load_op (ldop_code, mode, out_data, src, mem, model_rtx);
+
+  /* If necessary, calculate the data in memory after the update by redoing the
+     operation from values in registers.  */
+  if (!out_result)
+    return;
+
+  if (short_mode)
+    {
+      src = gen_lowpart (wmode, src);
+      out_data = gen_lowpart (wmode, out_data);
+      out_result = gen_lowpart (wmode, out_result);
+    }
+
+  x = NULL_RTX;
+
+  switch (code)
+    {
+    case MINUS:
+    case PLUS:
+      x = gen_rtx_PLUS (wmode, out_data, src);
+      break;
+    case IOR:
+      x = gen_rtx_IOR (wmode, out_data, src);
+      break;
+    case XOR:
+      x = gen_rtx_XOR (wmode, out_data, src);
+      break;
+    case AND:
+      aarch64_emit_bic (wmode, out_result, out_data, src, 0);
+      return;
+    default:
+      gcc_unreachable ();
+    }
+
+  emit_set_insn (out_result, x);
+
+  return;
 }
 
 /* Split an atomic operation.  */
diff --git a/gcc/config/aarch64/atomics.md b/gcc/config/aarch64/atomics.md
index e0d8856..e7ac5f6 100644
--- a/gcc/config/aarch64/atomics.md
+++ b/gcc/config/aarch64/atomics.md
@@ -219,7 +219,7 @@
   "&& reload_completed"
   [(const_int 0)]
   {
-    aarch64_gen_atomic_ldop (SET, operands[0], operands[1],
+    aarch64_gen_atomic_ldop (SET, operands[0], NULL, operands[1],
 			     operands[2], operands[3]);
     DONE;
   }
@@ -280,7 +280,7 @@
   "&& reload_completed"
   [(const_int 0)]
   {
-    aarch64_gen_atomic_ldop (<CODE>, operands[3], operands[0],
+    aarch64_gen_atomic_ldop (<CODE>, operands[3], NULL, operands[0],
 			     operands[1], operands[2]);
     DONE;
   }
@@ -368,7 +368,7 @@
   "&& reload_completed"
   [(const_int 0)]
   {
-    aarch64_gen_atomic_ldop (<CODE>, operands[0], operands[1],
+    aarch64_gen_atomic_ldop (<CODE>, operands[0], NULL, operands[1],
 			     operands[2], operands[3]);
     DONE;
   }
@@ -398,7 +398,31 @@
   }
 )
 
-(define_insn_and_split "atomic_<atomic_optab>_fetch<mode>"
+;; Load-operate-store, returning the original memory data.
+
+(define_expand "atomic_<atomic_optab>_fetch<mode>"
+ [(match_operand:ALLI 0 "register_operand" "")
+  (atomic_op:ALLI
+   (match_operand:ALLI 1 "aarch64_sync_memory_operand" "")
+   (match_operand:ALLI 2 "<atomic_op_operand>" ""))
+  (match_operand:SI 3 "const_int_operand")]
+ ""
+{
+  rtx (*gen) (rtx, rtx, rtx, rtx);
+  rtx value = operands[2];
+
+  /* Use an atomic load-operate instruction when possible.  */
+  if (aarch64_atomic_ldop_supported_p (<CODE>))
+    gen = gen_aarch64_atomic_<atomic_optab>_fetch<mode>_lse;
+  else
+    gen = gen_aarch64_atomic_<atomic_optab>_fetch<mode>;
+
+  emit_insn (gen (operands[0], operands[1], value, operands[3]));
+
+  DONE;
+})
+
+(define_insn_and_split "aarch64_atomic_<atomic_optab>_fetch<mode>"
   [(set (match_operand:ALLI 0 "register_operand" "=&r")
     (atomic_op:ALLI
       (match_operand:ALLI 1 "aarch64_sync_memory_operand" "+Q")
@@ -421,6 +445,29 @@
   }
 )
 
+(define_insn_and_split "aarch64_atomic_<atomic_optab>_fetch<mode>_lse"
+  [(set (match_operand:ALLI 0 "register_operand" "=&r")
+    (atomic_op:ALLI
+     (match_operand:ALLI 1 "aarch64_sync_memory_operand" "+Q")
+     (match_operand:ALLI 2 "<atomic_op_operand>" "r<const_atomic>")))
+   (set (match_dup 1)
+    (unspec_volatile:ALLI
+      [(match_dup 1)
+       (match_dup 2)
+       (match_operand:SI 3 "const_int_operand")]
+      UNSPECV_ATOMIC_LDOP))
+     (clobber (match_scratch:ALLI 4 "=r"))]
+  "TARGET_LSE"
+  "#"
+  "&& reload_completed"
+  [(const_int 0)]
+  {
+    aarch64_gen_atomic_ldop (<CODE>, operands[4], operands[0], operands[1],
+			     operands[2], operands[3]);
+    DONE;
+  }
+)
+
 (define_insn_and_split "atomic_nand_fetch<mode>"
   [(set (match_operand:ALLI 0 "register_operand" "=&r")
     (not:ALLI