re PR target/44452 (gcc.target/i386/abi-2.c and gcc.target/i386/pr22076.c fail on 32-bit Solaris 10+/x86)

	gcc/testsuite:
	PR target/44452
	* gcc.target/i386/pr22076.c: Add -mno-vect8-ret-in-mem on
	i?86-*-solaris2.[89], *-*-vxworks*.
	* gcc.target/i386/pr22152.c: Likewise.
	* gcc.target/i386/vect8-ret.c: New test.

	gcc:
	PR target/44452
	* config/i386/i386.opt (mvect8-ret-in-mem): Define.
	* config/i386/i386.c (ix86_target_string): Handle -mvect8-ret-in-mem.
	(ix86_solaris_return_in_memory): Remove.
	* config/i386/i386-protos.h (ix86_solaris_return_in_memory): Remove.
	* config/i386/sol2.h (SUBTARGET_RETURN_IN_MEMORY): Remove.
	(TARGET_SUBTARGET_DEFAULT): Redefine.
	* config/i386/sol2-10.h (TARGET_SUBTARGET_DEFAULT): Update comment.
	* config/i386/vx-common.h (SUBTARGET_RETURN_IN_MEMORY): Remove.
	(TARGET_SUBTARGET_DEFAULT): Redefine.
	* doc/invoke.texi (Option Summary, i386 and x86-64 Options): Add
	-mvect8-ret-in-mem.
	(i386 and x86-64 Options): Document -mvect8-ret-in-mem.

From-SVN: r164687

6 files changed, 24 insertions, 47 deletions

diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 700dec1..75562e8 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -140,7 +140,6 @@ extern rtx ix86_libcall_value (enum machine_mode);
 extern bool ix86_function_arg_regno_p (int);
 extern void ix86_asm_output_function_label (FILE *, const char *, tree);
 extern int ix86_function_arg_boundary (enum machine_mode, const_tree);
-extern bool ix86_solaris_return_in_memory (const_tree, const_tree);
 extern rtx ix86_force_to_memory (enum machine_mode, rtx);
 extern void ix86_free_from_memory (enum machine_mode);
 extern void ix86_call_abi_override (const_tree);
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 1d1c300..e164c4b 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -2632,6 +2632,7 @@ ix86_target_string (int isa, int flags, const char *arch, const char *tune,
     { "-msseregparm",			MASK_SSEREGPARM },
     { "-mstack-arg-probe",		MASK_STACK_PROBE },
     { "-mtls-direct-seg-refs",		MASK_TLS_DIRECT_SEG_REFS },
+    { "-mvect8-ret-in-mem",		MASK_VECT8_RETURNS },
     { "-m8bit-idiv",			MASK_USE_8BIT_IDIV },
   };
 
@@ -6849,9 +6850,9 @@ return_in_memory_32 (const_tree type, enum machine_mode mode)
 	return false;
 
       /* MMX/3dNow values are returned in MM0,
-	 except when it doesn't exits.  */
+	 except when it doesn't exits or the ABI prescribes otherwise.  */
       if (size == 8)
-	return !TARGET_MMX;
+	return !TARGET_MMX || TARGET_VECT8_RETURNS;
 
       /* SSE values are returned in XMM0, except when it doesn't exist.  */
       if (size == 16)
@@ -6915,43 +6916,6 @@ ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
 #endif
 }
 
-/* Return false iff TYPE is returned in memory.  This version is used
-   on Solaris 2.  It is similar to the generic ix86_return_in_memory,
-   but differs notably in that when MMX is available, 8-byte vectors
-   are returned in memory, rather than in MMX registers.  */
-
-bool
-ix86_solaris_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
-{
-  int size;
-  enum machine_mode mode = type_natural_mode (type, NULL);
-
-  if (TARGET_64BIT)
-    return return_in_memory_64 (type, mode);
-
-  if (mode == BLKmode)
-    return 1;
-
-  size = int_size_in_bytes (type);
-
-  if (VECTOR_MODE_P (mode))
-    {
-      /* Return in memory only if MMX registers *are* available.  This
-	 seems backwards, but it is consistent with the existing
-	 Solaris x86 ABI.  */
-      if (size == 8)
-	return TARGET_MMX;
-      if (size == 16)
-	return !TARGET_SSE;
-    }
-  else if (mode == TImode)
-    return !TARGET_SSE;
-  else if (mode == XFmode)
-    return 0;
-
-  return size > 12;
-}
-
 /* When returning SSE vector types, we have a choice of either
      (1) being abi incompatible with a -march switch, or
      (2) generating an error.
diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
index 38a53f6..09a4e06 100644
--- a/gcc/config/i386/i386.opt
+++ b/gcc/config/i386/i386.opt
@@ -244,6 +244,10 @@ mveclibabi=
 Target RejectNegative Joined Var(ix86_veclibabi_string)
 Vector library ABI to use
 
+mvect8-ret-in-mem
+Target Report Mask(VECT8_RETURNS) Save
+Return 8-byte vectors in memory
+
 mrecip
 Target Report Mask(RECIP) Save
 Generate reciprocals instead of divss and sqrtss.
diff --git a/gcc/config/i386/sol2-10.h b/gcc/config/i386/sol2-10.h
index c7fdec9..d4f6450 100644
--- a/gcc/config/i386/sol2-10.h
+++ b/gcc/config/i386/sol2-10.h
@@ -89,9 +89,12 @@ along with GCC; see the file COPYING3.  If not see
     }								\
   while (0)
 
+/* Override i386/sol2.h version: return 8-byte vectors in MMX registers if
+   possible, matching Sun Studio 12 Update 1+ compilers and other x86
+   targets.  */
 #undef TARGET_SUBTARGET_DEFAULT
-#define TARGET_SUBTARGET_DEFAULT (MASK_80387 | MASK_IEEE_FP	\
-				  | MASK_FLOAT_RETURNS)
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS)
 
 #define SUBTARGET_OPTIMIZATION_OPTIONS			\
   do							\
diff --git a/gcc/config/i386/sol2.h b/gcc/config/i386/sol2.h
index 57f75ea..b86308f 100644
--- a/gcc/config/i386/sol2.h
+++ b/gcc/config/i386/sol2.h
@@ -140,9 +140,15 @@ along with GCC; see the file COPYING3.  If not see
 /* Register the Solaris-specific #pragma directives.  */
 #define REGISTER_SUBTARGET_PRAGMAS() solaris_register_pragmas ()
 
+/* Undo i386/sysv4.h version.  */
 #undef SUBTARGET_RETURN_IN_MEMORY
-#define SUBTARGET_RETURN_IN_MEMORY(TYPE, FNTYPE) \
-	ix86_solaris_return_in_memory (TYPE, FNTYPE)
+
+/* Augment i386/unix.h version to return 8-byte vectors in memory, matching
+   Sun Studio compilers until version 12, the only ones supported on
+   Solaris 8 and 9.  */
+#undef TARGET_SUBTARGET_DEFAULT
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS | MASK_VECT8_RETURNS)
 
 /* Output a simple call for .init/.fini.  */
 #define ASM_OUTPUT_CALL(FILE, FN)				\
diff --git a/gcc/config/i386/vx-common.h b/gcc/config/i386/vx-common.h
index f4547f0..e63b91e 100644
--- a/gcc/config/i386/vx-common.h
+++ b/gcc/config/i386/vx-common.h
@@ -20,10 +20,11 @@ along with GCC; see the file COPYING3.  If not see
 #define ASM_OUTPUT_ALIGNED_BSS(FILE, DECL, NAME, SIZE, ALIGN) \
   asm_output_aligned_bss (FILE, DECL, NAME, SIZE, ALIGN)
 
-/* VxWorks uses the same ABI as Solaris 2.  */
+/* VxWorks uses the same ABI as Solaris 2, so use i386/sol2.h version.  */
 
-#define SUBTARGET_RETURN_IN_MEMORY(TYPE, FNTYPE) \
-	ix86_solaris_return_in_memory (TYPE, FNTYPE)
+#undef TARGET_SUBTARGET_DEFAULT
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS | MASK_VECT8_RETURNS)
 
 /* Provide our target specific DBX_REGISTER_NUMBER, as advertised by the
    common svr4.h.  VxWorks relies on the SVR4 numbering.  */