[gdb/tdep] Fix avx512 -m32 support in gdbserver

PR27257 reports a problem that can be reproduced as follows:
- use x86_64 machine with avx512 support
- compile a hello world with -m32 to a.out
- start a gdbserver session with a.out
- use gdb to connect to the gdbserver session

This makes us run into:
...
Listening on port 2346
Remote debugging from host ::1, port 34940
src/gdbserver/regcache.cc:257: \
  A problem internal to GDBserver has been detected.
Unknown register zmm16h requested
...

The problem is that i387_xsave_to_cache in gdbserver/i387-fp.cc can't find a
register zmm16h in the register cache.

To understand how this happens, first some background.

SSE has 16 128-bit wide xmm registers.

AVX extends the SSE registers set as follows:
- it extends the 16 existing 128-bit wide xmm registers to 256-bit wide ymm
  registers.

AVX512 extends the AVX register set as follows:
- it extends the 16 existing 256-bit wide ymm registers to 512-bit wide zmm
  registers.
- it adds 16 additional 512-bit wide zmm registers (with corresponding ymm and
  xmm subregisters added as well)

However, in 32-bit mode, there are only 8 xmm/ymm/zmm registers.

The problem we're running into is that gdbserver/i387-fp.cc uses these
constants to describe the size of the register file:
...
static const int num_avx512_zmmh_low_registers = 16;
static const int num_avx512_zmmh_high_registers = 16;
static const int num_avx512_ymmh_registers = 16;
static const int num_avx512_xmm_registers = 16;
...
which are all incorrect for the 32-bit case.

Fix this by replacing the constants with variables that have the appropriate
values in 64-bit and 32-bit mode.

Tested on x86_64-linux with native and unix/-m32.

1 files changed, 37 insertions, 13 deletions

diff --git a/gdbserver/i387-fp.cc b/gdbserver/i387-fp.cc
index 6d9b62e..9b07ded 100644
--- a/gdbserver/i387-fp.cc
+++ b/gdbserver/i387-fp.cc
@@ -23,10 +23,6 @@
 static const int num_mpx_bnd_registers = 4;
 static const int num_mpx_cfg_registers = 2;
 static const int num_avx512_k_registers = 8;
-static const int num_avx512_zmmh_low_registers = 16;
-static const int num_avx512_zmmh_high_registers = 16;
-static const int num_avx512_ymmh_registers = 16;
-static const int num_avx512_xmm_registers = 16;
 static const int num_pkeys_registers = 1;
 
 /* Note: These functions preserve the reserved bits in control registers.
@@ -256,14 +252,22 @@ void
 i387_cache_to_xsave (struct regcache *regcache, void *buf)
 {
   struct i387_xsave *fp = (struct i387_xsave *) buf;
+  bool amd64 = register_size (regcache->tdesc, 0) == 8;
   int i;
   unsigned long val, val2;
   unsigned long long xstate_bv = 0;
   unsigned long long clear_bv = 0;
   char raw[64];
   char *p;
+
   /* Amd64 has 16 xmm regs; I386 has 8 xmm regs.  */
-  int num_xmm_registers = register_size (regcache->tdesc, 0) == 8 ? 16 : 8;
+  int num_xmm_registers = amd64 ? 16 : 8;
+  /* AVX512 extends the existing xmm/ymm registers to a wider mode: zmm.  */
+  int num_avx512_zmmh_low_registers = num_xmm_registers;
+  /* AVX512 adds 16 extra regs in Amd64 mode, but none in I386 mode.*/
+  int num_avx512_zmmh_high_registers = amd64 ? 16 : 0;
+  int num_avx512_ymmh_registers = amd64 ? 16 : 0;
+  int num_avx512_xmm_registers = amd64 ? 16 : 0;
 
   /* The supported bits in `xstat_bv' are 8 bytes.  Clear part in
      vector registers if its bit in xstat_bv is zero.  */
@@ -452,7 +456,9 @@ i387_cache_to_xsave (struct regcache *regcache, void *buf)
   /* Check if any of ZMM16H-ZMM31H registers are changed.  */
   if ((x86_xcr0 & X86_XSTATE_ZMM))
     {
-      int zmm16h_regnum = find_regno (regcache->tdesc, "zmm16h");
+      int zmm16h_regnum = (num_avx512_zmmh_high_registers == 0
+			   ? -1
+			   : find_regno (regcache->tdesc, "zmm16h"));
 
       for (i = 0; i < num_avx512_zmmh_high_registers; i++)
 	{
@@ -469,7 +475,9 @@ i387_cache_to_xsave (struct regcache *regcache, void *buf)
   /* Check if any XMM_AVX512 registers are changed.  */
   if ((x86_xcr0 & X86_XSTATE_ZMM))
     {
-      int xmm_avx512_regnum = find_regno (regcache->tdesc, "xmm16");
+      int xmm_avx512_regnum = (num_avx512_xmm_registers == 0
+			       ? -1
+			       : find_regno (regcache->tdesc, "xmm16"));
 
       for (i = 0; i < num_avx512_xmm_registers; i++)
 	{
@@ -486,7 +494,9 @@ i387_cache_to_xsave (struct regcache *regcache, void *buf)
   /* Check if any YMMH_AVX512 registers are changed.  */
   if ((x86_xcr0 & X86_XSTATE_ZMM))
     {
-      int ymmh_avx512_regnum = find_regno (regcache->tdesc, "ymm16h");
+      int ymmh_avx512_regnum = (num_avx512_ymmh_registers == 0
+				? -1
+				: find_regno (regcache->tdesc, "ymm16h"));
 
       for (i = 0; i < num_avx512_ymmh_registers; i++)
 	{
@@ -710,12 +720,20 @@ i387_xsave_to_cache (struct regcache *regcache, const void *buf)
 {
   struct i387_xsave *fp = (struct i387_xsave *) buf;
   struct i387_fxsave *fxp = (struct i387_fxsave *) buf;
+  bool amd64 = register_size (regcache->tdesc, 0) == 8;
   int i, top;
   unsigned long val;
   unsigned long long clear_bv;
   gdb_byte *p;
-  /* Amd64 has 16 xmm regs; I386 has 8 xmm regs.  */
-  int num_xmm_registers = register_size (regcache->tdesc, 0) == 8 ? 16 : 8;
+
+   /* Amd64 has 16 xmm regs; I386 has 8 xmm regs.  */
+  int num_xmm_registers = amd64 ? 16 : 8;
+  /* AVX512 extends the existing xmm/ymm registers to a wider mode: zmm.  */
+  int num_avx512_zmmh_low_registers = num_xmm_registers;
+  /* AVX512 adds 16 extra regs in Amd64 mode, but none in I386 mode.*/
+  int num_avx512_zmmh_high_registers = amd64 ? 16 : 0;
+  int num_avx512_ymmh_registers = amd64 ? 16 : 0;
+  int num_avx512_xmm_registers = amd64 ? 16 : 0;
 
   /* The supported bits in `xstat_bv' are 8 bytes.  Clear part in
      vector registers if its bit in xstat_bv is zero.  */
@@ -845,9 +863,15 @@ i387_xsave_to_cache (struct regcache *regcache, const void *buf)
 
   if ((x86_xcr0 & X86_XSTATE_ZMM) != 0)
     {
-      int zmm16h_regnum = find_regno (regcache->tdesc, "zmm16h");
-      int ymm16h_regnum = find_regno (regcache->tdesc, "ymm16h");
-      int xmm16_regnum = find_regno (regcache->tdesc, "xmm16");
+      int zmm16h_regnum = (num_avx512_zmmh_high_registers == 0
+			   ? -1
+			   : find_regno (regcache->tdesc, "zmm16h"));
+      int ymm16h_regnum = (num_avx512_ymmh_registers == 0
+			   ? -1
+			   : find_regno (regcache->tdesc, "ymm16h"));
+      int xmm16_regnum = (num_avx512_xmm_registers == 0
+			  ? -1
+			  : find_regno (regcache->tdesc, "xmm16"));
 
       if ((clear_bv & X86_XSTATE_ZMM) != 0)
 	{