i386: Change indirect_return to function type attribute

In

struct ucontext;
typedef struct ucontext ucontext_t;

extern int (*bar) (ucontext_t *__restrict __oucp,
                   const ucontext_t *__restrict __ucp)
  __attribute__((__indirect_return__));

extern int res;

void
foo (ucontext_t *oucp, ucontext_t *ucp)
{
  res = bar (oucp, ucp);
}

bar() may return via indirect branch.  This patch changes indirect_return
to type attribute to allow indirect_return attribute on variable or type
of function pointer so that ENDBR can be inserted after call to bar().

gcc/

	PR target/86560
	* config/i386/i386.c (rest_of_insert_endbranch): Lookup
	indirect_return as function type attribute.
	(ix86_attribute_table): Change indirect_return to function
	type attribute.
	* doc/extend.texi: Update indirect_return attribute.

gcc/testsuite/

	PR target/86560
	* gcc.target/i386/pr86560-1.c: New test.
	* gcc.target/i386/pr86560-2.c: Likewise.
	* gcc.target/i386/pr86560-3.c: Likewise.

From-SVN: r262877

7 files changed, 83 insertions, 10 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index ecdaf32..78b8d39 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,12 @@
+2018-07-19  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/86560
+	* config/i386/i386.c (rest_of_insert_endbranch): Lookup
+	indirect_return as function type attribute.
+	(ix86_attribute_table): Change indirect_return to function
+	type attribute.
+	* doc/extend.texi: Update indirect_return attribute.
+
 2018-07-19  Aldy Hernandez  <aldyh@redhat.com>
 
 	* wide-int.h (widest2_int): New.
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 2b7e948..ee409cf 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -2635,16 +2635,23 @@ rest_of_insert_endbranch (void)
 		{
 		  rtx call = get_call_rtx_from (insn);
 		  rtx fnaddr = XEXP (call, 0);
+		  tree fndecl = NULL_TREE;
 
 		  /* Also generate ENDBRANCH for non-tail call which
 		     may return via indirect branch.  */
-		  if (MEM_P (fnaddr)
-		      && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+		  if (GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+		    fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
+		  if (fndecl == NULL_TREE)
+		    fndecl = MEM_EXPR (fnaddr);
+		  if (fndecl
+		      && TREE_CODE (TREE_TYPE (fndecl)) != FUNCTION_TYPE
+		      && TREE_CODE (TREE_TYPE (fndecl)) != METHOD_TYPE)
+		    fndecl = NULL_TREE;
+		  if (fndecl && TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
 		    {
-		      tree fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
-		      if (fndecl
-			  && lookup_attribute ("indirect_return",
-					       DECL_ATTRIBUTES (fndecl)))
+		      tree fntype = TREE_TYPE (fndecl);
+		      if (lookup_attribute ("indirect_return",
+					    TYPE_ATTRIBUTES (fntype)))
 			need_endbr = true;
 		    }
 		}
@@ -45920,8 +45927,8 @@ static const struct attribute_spec ix86_attribute_table[] =
     ix86_handle_fndecl_attribute, NULL },
   { "function_return", 1, 1, true, false, false, false,
     ix86_handle_fndecl_attribute, NULL },
-  { "indirect_return", 0, 0, true, false, false, false,
-    ix86_handle_fndecl_attribute, NULL },
+  { "indirect_return", 0, 0, false, true, true, false,
+    NULL, NULL },
 
   /* End element.  */
   { NULL, 0, 0, false, false, false, false, NULL, NULL }
diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
index abac85c..7b471ec 100644
--- a/gcc/doc/extend.texi
+++ b/gcc/doc/extend.texi
@@ -5889,8 +5889,9 @@ foo (void)
 @item indirect_return
 @cindex @code{indirect_return} function attribute, x86
 
-The @code{indirect_return} attribute on a function is used to inform
-the compiler that the function may return via indirect branch.
+The @code{indirect_return} attribute can be applied to a function,
+as well as variable or type of function pointer to inform the
+compiler that the function may return via indirect branch.
 
 @end table
 
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index f78a7c8..8c60524 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,10 @@
+2018-07-19  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/86560
+	* gcc.target/i386/pr86560-1.c: New test.
+	* gcc.target/i386/pr86560-2.c: Likewise.
+	* gcc.target/i386/pr86560-3.c: Likewise.
+
 2018-07-19  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>
 
 	* gfortran.dg/max_fmax_aarch64.f90: New test.
diff --git a/gcc/testsuite/gcc.target/i386/pr86560-1.c b/gcc/testsuite/gcc.target/i386/pr86560-1.c
new file mode 100644
index 0000000..a2b70269
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr86560-1.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+struct ucontext;
+
+extern int (*bar) (struct ucontext *)
+  __attribute__((__indirect_return__));
+
+extern int res;
+
+void
+foo (struct ucontext *oucp)
+{
+  res = bar (oucp);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr86560-2.c b/gcc/testsuite/gcc.target/i386/pr86560-2.c
new file mode 100644
index 0000000..6f01b38
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr86560-2.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+struct ucontext;
+
+typedef int (*bar_p) (struct ucontext *)
+  __attribute__((__indirect_return__));
+
+extern int res;
+
+void
+foo (bar_p bar, struct ucontext *oucp)
+{
+  res = bar (oucp);
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr86560-3.c b/gcc/testsuite/gcc.target/i386/pr86560-3.c
new file mode 100644
index 0000000..05328e2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr86560-3.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+struct ucontext;
+
+extern int (*bar) (struct ucontext *);
+
+extern int res;
+
+void
+foo (struct ucontext *oucp)
+{
+  int (*f) (struct ucontext *) __attribute__((__indirect_return__))
+    = bar;
+  res = f (oucp);
+}