AArch64 - use CSDB based sequences if speculation tracking is enabled

In this final patch, now that we can track speculation through conditional
branches, we can use this information to use a less expensive CSDB based
speculation barrier.

	* config/aarch64/iterators.md (ALLI_TI): New iterator.
	* config/aarch64/aarch64.md (despeculate_copy<ALLI_TI:mode>): New
	expand.
	(despeculate_copy<ALLI:mode>_insn): New insn.
	(despeculate_copyti_insn): New insn.
	(despeculate_simple<ALLI:mode>): New insn
	(despeculate_simpleti): New insn.
	* config/aarch64/aarch64.c (aarch64_speculation_safe_value): New
	function.
	(TARGET_SPECULATION_SAFE_VALUE): Redefine to
	aarch64_speculation_safe_value.
	(aarch64_print_operand): Handle const0_rtx in modifier 'H'.

From-SVN: r263174

4 files changed, 163 insertions, 0 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index c21608a..48a2002 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,5 +1,20 @@
 2018-07-31  Richard Earnshaw  <rearnsha@arm.com>
 
+	* config/aarch64/iterators.md (ALLI_TI): New iterator.
+	* config/aarch64/aarch64.md (despeculate_copy<ALLI_TI:mode>): New
+	expand.
+	(despeculate_copy<ALLI:mode>_insn): New insn.
+	(despeculate_copyti_insn): New insn.
+	(despeculate_simple<ALLI:mode>): New insn
+	(despeculate_simpleti): New insn.
+	* config/aarch64/aarch64.c (aarch64_speculation_safe_value): New
+	function.
+	(TARGET_SPECULATION_SAFE_VALUE): Redefine to
+	aarch64_speculation_safe_value.
+	(aarch64_print_operand): Handle const0_rtx in modifier 'H'.
+
+2018-07-31  Richard Earnshaw  <rearnsha@arm.com>
+
 	* config/aarch64/aarch64-speculation.cc: New file.
 	* config/aarch64/aarch64-passes.def (pass_track_speculation): Add
 	before pass_reorder_blocks.
diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index dc8cd2b..f743f57 100644
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -6785,6 +6785,12 @@ aarch64_print_operand (FILE *f, rtx x, int code)
       break;
 
     case 'H':
+      if (x == const0_rtx)
+	{
+	  asm_fprintf (f, "xzr");
+	  break;
+	}
+
       if (!REG_P (x) || !GP_REGNUM_P (REGNO (x) + 1))
 	{
 	  output_operand_lossage ("invalid operand for '%%%c'", code);
@@ -17778,6 +17784,45 @@ aarch64_select_early_remat_modes (sbitmap modes)
     }
 }
 
+/* Override the default target speculation_safe_value.  */
+static rtx
+aarch64_speculation_safe_value (machine_mode mode,
+				rtx result, rtx val, rtx failval)
+{
+  /* Maybe we should warn if falling back to hard barriers.  They are
+     likely to be noticably more expensive than the alternative below.  */
+  if (!aarch64_track_speculation)
+    return default_speculation_safe_value (mode, result, val, failval);
+
+  if (!REG_P (val))
+    val = copy_to_mode_reg (mode, val);
+
+  if (!aarch64_reg_or_zero (failval, mode))
+    failval = copy_to_mode_reg (mode, failval);
+
+  switch (mode)
+    {
+    case E_QImode:
+      emit_insn (gen_despeculate_copyqi (result, val, failval));
+      break;
+    case E_HImode:
+      emit_insn (gen_despeculate_copyhi (result, val, failval));
+      break;
+    case E_SImode:
+      emit_insn (gen_despeculate_copysi (result, val, failval));
+      break;
+    case E_DImode:
+      emit_insn (gen_despeculate_copydi (result, val, failval));
+      break;
+    case E_TImode:
+      emit_insn (gen_despeculate_copyti (result, val, failval));
+      break;
+    default:
+      gcc_unreachable ();
+    }
+  return result;
+}
+
 /* Target-specific selftests.  */
 
 #if CHECKING_P
@@ -18250,6 +18295,9 @@ aarch64_libgcc_floating_mode_supported_p
 #undef TARGET_SELECT_EARLY_REMAT_MODES
 #define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
 
+#undef TARGET_SPECULATION_SAFE_VALUE
+#define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value
+
 #if CHECKING_P
 #undef TARGET_RUN_TARGET_SELFTESTS
 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests
diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 3dcc054..9f3cade 100644
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -6590,6 +6590,103 @@
    (set_attr "speculation_barrier" "true")]
 )
 
+;; Support for __builtin_speculation_safe_value when we have speculation
+;; tracking enabled.  Use the speculation tracker to decide whether to
+;; copy operand 1 to the target, or to copy the fail value (operand 2).
+(define_expand "despeculate_copy<ALLI_TI:mode>"
+  [(set (match_operand:ALLI_TI 0 "register_operand" "=r")
+	(unspec_volatile:ALLI_TI
+	 [(match_operand:ALLI_TI 1 "register_operand" "r")
+	  (match_operand:ALLI_TI 2 "aarch64_reg_or_zero" "rZ")
+	  (use (reg:DI SPECULATION_TRACKER_REGNUM))
+	  (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+  ""
+  "
+  {
+    if (operands[2] == const0_rtx)
+      {
+	rtx tracker;
+	if (<MODE>mode == TImode)
+	  tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+	else
+	  tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
+
+	emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
+						 tracker));
+	DONE;
+      }
+  }
+  "
+)
+
+;; Patterns to match despeculate_copy<mode>.  Note that "hint 0x14" is the
+;; encoding for CSDB, but will work in older versions of the assembler.
+(define_insn "*despeculate_copy<ALLI:mode>_insn"
+  [(set (match_operand:ALLI 0 "register_operand" "=r")
+	(unspec_volatile:ALLI
+	 [(match_operand:ALLI 1 "register_operand" "r")
+	  (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
+	  (use (reg:DI SPECULATION_TRACKER_REGNUM))
+	  (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+  ""
+  {
+    operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+    output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
+		     operands);
+    return "";
+  }
+  [(set_attr "length" "12")
+   (set_attr "type" "block")
+   (set_attr "speculation_barrier" "true")]
+)
+
+;; Pattern to match despeculate_copyti
+(define_insn "*despeculate_copyti_insn"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(unspec_volatile:TI
+	 [(match_operand:TI 1 "register_operand" "r")
+	  (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
+	  (use (reg:DI SPECULATION_TRACKER_REGNUM))
+	  (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+  ""
+  {
+    operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+    output_asm_insn
+      ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
+       operands);
+    return "";
+  }
+  [(set_attr "length" "16")
+   (set_attr "type" "block")
+   (set_attr "speculation_barrier" "true")]
+)
+
+(define_insn "despeculate_simple<ALLI:mode>"
+  [(set (match_operand:ALLI 0 "register_operand" "=r")
+	(unspec_volatile:ALLI
+	 [(match_operand:ALLI 1 "register_operand" "r")
+	  (use (match_operand:ALLI 2 "register_operand" ""))]
+	 UNSPECV_SPECULATION_BARRIER))]
+  ""
+  "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
+  [(set_attr "type" "block")
+   (set_attr "length" "8")
+   (set_attr "speculation_barrier" "true")]
+)
+
+(define_insn "despeculate_simpleti"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(unspec_volatile:TI
+	 [(match_operand:TI 1 "register_operand" "r")
+	  (use (match_operand:DI 2 "register_operand" ""))]
+	 UNSPECV_SPECULATION_BARRIER))]
+  ""
+  "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
+  [(set_attr "type" "block")
+   (set_attr "length" "12")
+   (set_attr "speculation_barrier" "true")]
+)
+
 ;; AdvSIMD Stuff
 (include "aarch64-simd.md")
 
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index d846118..450edea 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -35,6 +35,9 @@
 ;; Iterator for all integer modes (up to 64-bit)
 (define_mode_iterator ALLI [QI HI SI DI])
 
+;; Iterator for all integer modes (up to 128-bit)
+(define_mode_iterator ALLI_TI [QI HI SI DI TI])
+
 ;; Iterator for all integer modes that can be extended (up to 64-bit)
 (define_mode_iterator ALLX [QI HI SI])