x86: Support x32 and IBT in heap trampoline

Add x32 and IBT support to x86 heap trampoline implementation with a
testcase.

2024-02-13  Jakub Jelinek  <jakub@redhat.com>
	    H.J. Lu  <hjl.tools@gmail.com>

libgcc/

	PR target/113855
	* config/i386/heap-trampoline.c (trampoline_insns): Add IBT
	support and pad to the multiple of 4 bytes.  Use movabsq
	instead of movabs in comments.  Add -mx32 variant.

gcc/testsuite/

	PR target/113855
	* gcc.dg/heap-trampoline-1.c: New test.
	* lib/target-supports.exp (check_effective_target_heap_trampoline):
	New.

3 files changed, 74 insertions, 3 deletions

diff --git a/gcc/testsuite/gcc.dg/heap-trampoline-1.c b/gcc/testsuite/gcc.dg/heap-trampoline-1.c
new file mode 100644
index 0000000..1aebe00
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/heap-trampoline-1.c
@@ -0,0 +1,23 @@
+/* { dg-do run { target heap_trampoline } } */
+/* { dg-options "-ftrampoline-impl=heap" } */
+
+__attribute__((noipa)) int
+bar (int (*fn) (int))
+{
+  return fn (42) + 1;
+}
+
+int
+main ()
+{
+  int a = 0;
+  int foo (int x) { if (x != 42) __builtin_abort (); return ++a; }
+  if (bar (foo) != 2 || a != 1)
+    __builtin_abort ();
+  if (bar (foo) != 3 || a != 2)
+    __builtin_abort ();
+  a = 42;
+  if (bar (foo) != 44 || a != 43)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index 6ce8557..8171599 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -13477,3 +13477,15 @@ proc dg-require-python-h { args } {
     eval lappend extra-tool-flags $python_flags
     verbose "After appending, extra-tool-flags: ${extra-tool-flags}" 3
 }
+
+# Return 1 if the target supports heap-trampoline, 0 otherwise.
+proc check_effective_target_heap_trampoline {} {
+    if { [istarget aarch64*-*-linux*]
+	 || [istarget i?86-*-darwin*]
+	 || [istarget x86_64-*-darwin*]
+	 || [istarget i?86-*-linux*]
+	 || [istarget x86_64-*-linux*] } {
+	return 1
+    }
+    return 0
+}
diff --git a/libgcc/config/i386/heap-trampoline.c b/libgcc/config/i386/heap-trampoline.c
index 1df0aa0..a8637dc 100644
--- a/libgcc/config/i386/heap-trampoline.c
+++ b/libgcc/config/i386/heap-trampoline.c
@@ -30,28 +30,64 @@ void __gcc_nested_func_ptr_created (void *chain, void *func, void *dst);
 void __gcc_nested_func_ptr_deleted (void);
 
 #if __x86_64__
+
+#ifdef __LP64__
 static const uint8_t trampoline_insns[] = {
-  /* movabs $<func>,%r11  */
+#if defined __CET__ && (__CET__ & 1) != 0
+  /* endbr64.  */
+  0xf3, 0x0f, 0x1e, 0xfa,
+#endif
+
+  /* movabsq $<func>,%r11  */
   0x49, 0xbb,
   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 
-  /* movabs $<chain>,%r10  */
+  /* movabsq $<chain>,%r10  */
   0x49, 0xba,
   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 
   /* rex.WB jmpq *%r11  */
-  0x41, 0xff, 0xe3
+  0x41, 0xff, 0xe3,
+
+  /* Pad to the multiple of 4 bytes.  */
+  0x90
 };
+#else
+static const uint8_t trampoline_insns[] = {
+#if defined __CET__ && (__CET__ & 1) != 0
+  /* endbr64.  */
+  0xf3, 0x0f, 0x1e, 0xfa,
+#endif
+
+  /* movl $<func>,%r11d  */
+  0x41, 0xbb,
+  0x00, 0x00, 0x00, 0x00,
+
+  /* movl $<chain>,%r10d  */
+  0x41, 0xba,
+  0x00, 0x00, 0x00, 0x00,
+
+  /* rex.WB jmpq *%r11  */
+  0x41, 0xff, 0xe3,
+
+  /* Pad to the multiple of 4 bytes.  */
+  0x90
+};
+#endif
 
 union ix86_trampoline {
   uint8_t insns[sizeof(trampoline_insns)];
 
   struct __attribute__((packed)) fields {
+#if defined __CET__ && (__CET__ & 1) != 0
+    uint8_t endbr64[4];
+#endif
     uint8_t insn_0[2];
     void *func_ptr;
     uint8_t insn_1[2];
     void *chain_ptr;
     uint8_t insn_2[3];
+    uint8_t pad;
   } fields;
 };