aarch64: Update sibcall handling for SME

We only support tail calls between functions with the same PSTATE.ZA
setting ("private-ZA" to "private-ZA" and "shared-ZA" to "shared-ZA").

Only a normal non-streaming function can tail-call another non-streaming
function, and only a streaming function can tail-call another streaming
function.  Any function can tail-call a streaming-compatible function.

gcc/
	* config/aarch64/aarch64.cc (aarch64_function_ok_for_sibcall):
	Enforce PSTATE.SM and PSTATE.ZA restrictions.
	(aarch64_expand_epilogue): Save and restore the arguments
	to a sibcall around any change to PSTATE.SM.

gcc/testsuite/
	* gcc.target/aarch64/sme/sibcall_1.c: New test.
	* gcc.target/aarch64/sme/sibcall_2.c: Likewise.
	* gcc.target/aarch64/sme/sibcall_3.c: Likewise.
	* gcc.target/aarch64/sme/sibcall_4.c: Likewise.
	* gcc.target/aarch64/sme/sibcall_5.c: Likewise.
	* gcc.target/aarch64/sme/sibcall_6.c: Likewise.
	* gcc.target/aarch64/sme/sibcall_7.c: Likewise.
	* gcc.target/aarch64/sme/sibcall_8.c: Likewise.

9 files changed, 304 insertions, 1 deletions

diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 4639310..48b7811 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -6124,6 +6124,11 @@ aarch64_function_ok_for_sibcall (tree, tree exp)
   if (crtl->abi->id () != expr_callee_abi (exp).id ())
     return false;
 
+  tree fntype = TREE_TYPE (TREE_TYPE (CALL_EXPR_FN (exp)));
+  if (aarch64_fntype_pstate_sm (fntype) & ~aarch64_cfun_incoming_pstate_sm ())
+    return false;
+  if (aarch64_fntype_pstate_za (fntype) != aarch64_cfun_incoming_pstate_za ())
+    return false;
   return true;
 }
 
@@ -9564,7 +9569,9 @@ aarch64_expand_epilogue (rtx_call_insn *sibcall)
 	guard_label = aarch64_guard_switch_pstate_sm (IP0_REGNUM,
 						      aarch64_isa_flags);
       aarch64_sme_mode_switch_regs return_switch;
-      if (crtl->return_rtx && REG_P (crtl->return_rtx))
+      if (sibcall)
+	return_switch.add_call_args (sibcall);
+      else if (crtl->return_rtx && REG_P (crtl->return_rtx))
 	return_switch.add_reg (GET_MODE (crtl->return_rtx),
 			       REGNO (crtl->return_rtx));
       return_switch.emit_prologue ();
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/sibcall_1.c b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_1.c
new file mode 100644
index 0000000..c7530de
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_1.c
@@ -0,0 +1,45 @@
+/* { dg-options "-O2" } */
+
+void sc_callee () [[arm::streaming_compatible]];
+void s_callee () [[arm::streaming]];
+void n_callee ();
+
+[[arm::locally_streaming]] __attribute__((noipa)) void
+sc_ls_callee () [[arm::streaming_compatible]] {}
+[[arm::locally_streaming]] __attribute__((noipa)) void
+n_ls_callee () {}
+
+void
+sc_to_sc () [[arm::streaming_compatible]]
+{
+  sc_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_callee} } } */
+
+void
+sc_to_s () [[arm::streaming_compatible]]
+{
+  s_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\ts_callee} } } */
+
+void
+sc_to_n () [[arm::streaming_compatible]]
+{
+  n_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tn_callee} } } */
+
+void
+sc_to_sc_ls () [[arm::streaming_compatible]]
+{
+  sc_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_ls_callee} } } */
+
+void
+sc_to_n_ls () [[arm::streaming_compatible]]
+{
+  n_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tn_ls_callee} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/sibcall_2.c b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_2.c
new file mode 100644
index 0000000..8d1c8a9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_2.c
@@ -0,0 +1,45 @@
+/* { dg-options "-O2" } */
+
+void sc_callee () [[arm::streaming_compatible]];
+void s_callee () [[arm::streaming]];
+void n_callee ();
+
+[[arm::locally_streaming]] __attribute__((noipa)) void
+sc_ls_callee () [[arm::streaming_compatible]] {}
+[[arm::locally_streaming]] __attribute__((noipa)) void
+n_ls_callee () {}
+
+void
+s_to_sc () [[arm::streaming]]
+{
+  sc_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_callee} } } */
+
+void
+s_to_s () [[arm::streaming]]
+{
+  s_callee ();
+}
+/* { dg-final { scan-assembler {\tb\ts_callee} } } */
+
+void
+s_to_n () [[arm::streaming]]
+{
+  n_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tn_callee} } } */
+
+void
+s_to_sc_ls () [[arm::streaming]]
+{
+  sc_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_ls_callee} } } */
+
+void
+s_to_n_ls () [[arm::streaming]]
+{
+  n_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tn_ls_callee} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/sibcall_3.c b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_3.c
new file mode 100644
index 0000000..2ae937f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_3.c
@@ -0,0 +1,45 @@
+/* { dg-options "-O2" } */
+
+void sc_callee () [[arm::streaming_compatible]];
+void s_callee () [[arm::streaming]];
+void n_callee ();
+
+[[arm::locally_streaming]] __attribute__((noipa)) void
+sc_ls_callee () [[arm::streaming_compatible]] {}
+[[arm::locally_streaming]] __attribute__((noipa)) void
+n_ls_callee () {}
+
+void
+n_to_sc ()
+{
+  sc_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_callee} } } */
+
+void
+n_to_s ()
+{
+  s_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\ts_callee} } } */
+
+void
+n_to_n ()
+{
+  n_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tn_callee} } } */
+
+void
+n_to_sc_ls ()
+{
+  sc_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_ls_callee} } } */
+
+void
+n_to_n_ls ()
+{
+  n_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tn_ls_callee} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/sibcall_4.c b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_4.c
new file mode 100644
index 0000000..6935a1b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_4.c
@@ -0,0 +1,45 @@
+/* { dg-options "-O2" } */
+
+void sc_callee () [[arm::streaming_compatible]];
+void s_callee () [[arm::streaming]];
+void n_callee ();
+
+[[arm::locally_streaming]] __attribute__((noipa)) void
+sc_ls_callee () [[arm::streaming_compatible]] {}
+[[arm::locally_streaming]] __attribute__((noipa)) void
+n_ls_callee () {}
+
+[[arm::locally_streaming]] void
+sc_to_sc () [[arm::streaming_compatible]]
+{
+  sc_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_callee} } } */
+
+[[arm::locally_streaming]] void
+sc_to_s () [[arm::streaming_compatible]]
+{
+  s_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\ts_callee} } } */
+
+[[arm::locally_streaming]] void
+sc_to_n () [[arm::streaming_compatible]]
+{
+  n_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tn_callee} } } */
+
+[[arm::locally_streaming]] void
+sc_to_sc_ls () [[arm::streaming_compatible]]
+{
+  sc_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_ls_callee} } } */
+
+[[arm::locally_streaming]] void
+sc_to_n_ls () [[arm::streaming_compatible]]
+{
+  n_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tn_ls_callee} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/sibcall_5.c b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_5.c
new file mode 100644
index 0000000..7aaf58d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_5.c
@@ -0,0 +1,45 @@
+/* { dg-options "-O2" } */
+
+void sc_callee () [[arm::streaming_compatible]];
+void s_callee () [[arm::streaming]];
+void n_callee ();
+
+[[arm::locally_streaming]] __attribute__((noipa)) void
+sc_ls_callee () [[arm::streaming_compatible]] {}
+[[arm::locally_streaming]] __attribute__((noipa)) void
+n_ls_callee () {}
+
+[[arm::locally_streaming]] void
+n_to_sc ()
+{
+  sc_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_callee} } } */
+
+[[arm::locally_streaming]] void
+n_to_s ()
+{
+  s_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\ts_callee} } } */
+
+[[arm::locally_streaming]] void
+n_to_n ()
+{
+  n_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tn_callee} } } */
+
+[[arm::locally_streaming]] void
+n_to_sc_ls ()
+{
+  sc_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tsc_ls_callee} } } */
+
+[[arm::locally_streaming]] void
+n_to_n_ls ()
+{
+  n_ls_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tn_ls_callee} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/sibcall_6.c b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_6.c
new file mode 100644
index 0000000..e568edb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_6.c
@@ -0,0 +1,26 @@
+/* { dg-options "-O2" } */
+
+void shared_callee () [[arm::inout("za")]];
+[[arm::new("za")]] __attribute__((noipa)) void new_callee () {}
+void normal_callee ();
+
+void
+shared_to_shared () [[arm::inout("za")]]
+{
+  shared_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tshared_callee} } } */
+
+void
+shared_to_new () [[arm::inout("za")]]
+{
+  new_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tnew_callee} } } */
+
+void
+shared_to_normal () [[arm::inout("za")]]
+{
+  normal_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tnormal_callee} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/sibcall_7.c b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_7.c
new file mode 100644
index 0000000..a5f576d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_7.c
@@ -0,0 +1,26 @@
+/* { dg-options "-O2" } */
+
+void shared_callee () [[arm::inout("za")]];
+[[arm::new("za")]] __attribute__((noipa)) void new_callee () {}
+void normal_callee ();
+
+[[arm::new("za")]] void
+new_to_shared ()
+{
+  shared_callee ();
+}
+/* { dg-final { scan-assembler {\tbl\tshared_callee} } } */
+
+[[arm::new("za")]] void
+new_to_new ()
+{
+  new_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tnew_callee} } } */
+
+[[arm::new("za")]] void
+new_to_normal ()
+{
+  normal_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tnormal_callee} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/sibcall_8.c b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_8.c
new file mode 100644
index 0000000..33370f7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/sibcall_8.c
@@ -0,0 +1,19 @@
+/* { dg-options "-O2" } */
+
+void shared_callee () [[arm::inout("za")]];
+[[arm::new("za")]] __attribute__((noipa)) void new_callee () {}
+void normal_callee ();
+
+void
+normal_to_new ()
+{
+  new_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tnew_callee} } } */
+
+void
+normal_to_normal ()
+{
+  normal_callee ();
+}
+/* { dg-final { scan-assembler {\tb\tnormal_callee} } } */