PR 19051: support of inferior call with gnu vector support on ARM

This patch teaches GDB to support gnu vector in inferior calls.  As a
result, fails in gdb.base/gnu_vector.exp are fixed.  The calling
convention of gnu vector isn't documented in the AAPCS, because it
is the GCC extension.  I checked the gcc/config/arm/arm.c, understand
how GCC pass arguments and return values, and do the same in GDB side.

The patch is tested with both hard float and soft float on arm-linux.

gdb:

2015-11-13  Yao Qi  <yao.qi@linaro.org>

	PR tdep/19051
	* arm-tdep.c (arm_type_align): Return the right alignment
	value for vector.
	(arm_vfp_cprc_sub_candidate): Return true for 64-bit and
	128-bit vector types.
	(arm_return_in_memory): Handel vector type.

2 files changed, 63 insertions, 14 deletions

diff --git a/gdb/ChangeLog b/gdb/ChangeLog
index a4e73e0..e48ffa1 100644
--- a/gdb/ChangeLog
+++ b/gdb/ChangeLog
@@ -1,5 +1,14 @@
 2015-11-13  Yao Qi  <yao.qi@linaro.org>
 
+	PR tdep/19051
+	* arm-tdep.c (arm_type_align): Return the right alignment
+	value for vector.
+	(arm_vfp_cprc_sub_candidate): Return true for 64-bit and
+	128-bit vector types.
+	(arm_return_in_memory): Handel vector type.
+
+2015-11-13  Yao Qi  <yao.qi@linaro.org>
+
 	* arm-tdep.c (arm_return_in_memory): Rewrite it.
 	(arm_return_value): Call arm_return_in_memory for
 	TYPE_CODE_COMPLEX.
diff --git a/gdb/arm-tdep.c b/gdb/arm-tdep.c
index c8b665a..b8f84ce 100644
--- a/gdb/arm-tdep.c
+++ b/gdb/arm-tdep.c
@@ -3446,8 +3446,18 @@ arm_type_align (struct type *t)
       return TYPE_LENGTH (t);
 
     case TYPE_CODE_ARRAY:
+      if (TYPE_VECTOR (t))
+	{
+	  /* Use the natural alignment for vector types (the same for
+	     scalar type), but the maximum alignment is 64-bit.  */
+	  if (TYPE_LENGTH (t) > 8)
+	    return 8;
+	  else
+	    return TYPE_LENGTH (t);
+	}
+      else
+	return arm_type_align (TYPE_TARGET_TYPE (t));
     case TYPE_CODE_COMPLEX:
-      /* TODO: What about vector types?  */
       return arm_type_align (TYPE_TARGET_TYPE (t));
 
     case TYPE_CODE_STRUCT:
@@ -3594,21 +3604,44 @@ arm_vfp_cprc_sub_candidate (struct type *t,
 
     case TYPE_CODE_ARRAY:
       {
-	int count;
-	unsigned unitlen;
-	count = arm_vfp_cprc_sub_candidate (TYPE_TARGET_TYPE (t), base_type);
-	if (count == -1)
-	  return -1;
-	if (TYPE_LENGTH (t) == 0)
+	if (TYPE_VECTOR (t))
 	  {
-	    gdb_assert (count == 0);
-	    return 0;
+	    /* A 64-bit or 128-bit containerized vector type are VFP
+	       CPRCs.  */
+	    switch (TYPE_LENGTH (t))
+	      {
+	      case 8:
+		if (*base_type == VFP_CPRC_UNKNOWN)
+		  *base_type = VFP_CPRC_VEC64;
+		return 1;
+	      case 16:
+		if (*base_type == VFP_CPRC_UNKNOWN)
+		  *base_type = VFP_CPRC_VEC128;
+		return 1;
+	      default:
+		return -1;
+	      }
+	  }
+	else
+	  {
+	    int count;
+	    unsigned unitlen;
+
+	    count = arm_vfp_cprc_sub_candidate (TYPE_TARGET_TYPE (t),
+						base_type);
+	    if (count == -1)
+	      return -1;
+	    if (TYPE_LENGTH (t) == 0)
+	      {
+		gdb_assert (count == 0);
+		return 0;
+	      }
+	    else if (count == 0)
+	      return -1;
+	    unitlen = arm_vfp_cprc_unit_length (*base_type);
+	    gdb_assert ((TYPE_LENGTH (t) % unitlen) == 0);
+	    return TYPE_LENGTH (t) / unitlen;
 	  }
-	else if (count == 0)
-	  return -1;
-	unitlen = arm_vfp_cprc_unit_length (*base_type);
-	gdb_assert ((TYPE_LENGTH (t) % unitlen) == 0);
-	return TYPE_LENGTH (t) / unitlen;
       }
       break;
 
@@ -9000,6 +9033,13 @@ arm_return_in_memory (struct gdbarch *gdbarch, struct type *type)
       && TYPE_CODE_ARRAY != code && TYPE_CODE_COMPLEX != code)
     return 0;
 
+  if (TYPE_CODE_ARRAY == code && TYPE_VECTOR (type))
+    {
+      /* Vector values should be returned using ARM registers if they
+	 are not over 16 bytes.  */
+      return (TYPE_LENGTH (type) > 16);
+    }
+
   if (gdbarch_tdep (gdbarch)->arm_abi != ARM_ABI_APCS)
     {
       /* The AAPCS says all aggregates not larger than a word are returned