x86: limit RegRex64 use

The special property really only applies to the "extended" byte regs
having legacy word/dword counterparts.

While touching involved code also drop redundant byte checks from a
conditional in establish_rex(): The other remaining RegRex64 uses only
exist on registers which can't be used as register operands anyway.
Hence RegRex64 as an attribute of a (valid) register operand implies
that it's a byte reg.

3 files changed, 55 insertions, 57 deletions

diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index ca87afd..a9a0c3f 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -4575,9 +4575,9 @@ static void establish_rex (void)
   i.rex |= i.prefix[REX_PREFIX] & REX_OPCODE;
 
   /* For 8 bit RegRex64 registers without a prefix, we need an empty rex prefix.  */
-  if (((i.types[first].bitfield.class == Reg && i.types[first].bitfield.byte
+  if (((i.types[first].bitfield.class == Reg
 	&& (i.op[first].regs->reg_flags & RegRex64) != 0)
-       || (i.types[last].bitfield.class == Reg && i.types[last].bitfield.byte
+       || (i.types[last].bitfield.class == Reg
 	   && (i.op[last].regs->reg_flags & RegRex64) != 0))
       && !is_apx_rex2_encoding () && !is_any_vex_encoding (&i.tm))
     i.rex |= REX_OPCODE;
@@ -4591,9 +4591,8 @@ static void establish_rex (void)
 	{
 	  /* Look for 8 bit operand that uses old registers.  */
 	  if (i.types[x].bitfield.class == Reg && i.types[x].bitfield.byte
-	      && (i.op[x].regs->reg_flags & RegRex64) == 0)
+	      && !(i.op[x].regs->reg_flags & (RegRex | RegRex2 | RegRex64)))
 	    {
-	      gas_assert (!(i.op[x].regs->reg_flags & RegRex));
 	      /* In case it is "hi" register, give up.  */
 	      if (i.op[x].regs->reg_num > 3)
 		as_bad (_("can't encode register '%s%s' in an "
@@ -4619,10 +4618,9 @@ static void establish_rex (void)
       for (x = first; x <= last; x++)
 	if (i.types[x].bitfield.class == Reg
 	    && i.types[x].bitfield.byte
-	    && (i.op[x].regs->reg_flags & RegRex64) == 0
+	    && !(i.op[x].regs->reg_flags & (RegRex | RegRex2 | RegRex64))
 	    && i.op[x].regs->reg_num > 3)
 	  {
-	    gas_assert (!(i.op[x].regs->reg_flags & RegRex));
 	    pp.rex_encoding = false;
 	    pp.rex2_encoding = false;
 	    break;
@@ -4927,7 +4925,7 @@ optimize_encoding (void)
 	  /* Squash the suffix.  */
 	  i.suffix = 0;
 	  /* Convert to byte registers. 8-bit registers are special,
-	     RegRex64 and non-RegRex64 each have 8 registers.  */
+	     RegRex64 and non-RegRex* each have 8 registers.  */
 	  if (i.types[1].bitfield.word)
 	    /* 32 (or 40) 8-bit registers.  */
 	    j = 32;
@@ -5480,7 +5478,7 @@ static bool is_index (const reg_entry *r)
 
   if (r->reg_type.bitfield.byte)
     {
-      if (!(r->reg_flags & RegRex64))
+      if (!(r->reg_flags & (RegRex | RegRex2 | RegRex64)))
 	{
 	  if (r->reg_num >= 4)
 	    return false;
@@ -13147,7 +13145,7 @@ s_insn (int dummy ATTRIBUTE_UNUSED)
 	      && flag_code == CODE_64BIT
 	      && i.types[j].bitfield.class == Reg
 	      && i.types[j].bitfield.byte
-	      && !(i.op[j].regs->reg_flags & RegRex64)
+	      && !(i.op[j].regs->reg_flags & (RegRex | RegRex2 | RegRex64))
 	      && i.op[j].regs->reg_num > 3)
 	    as_bad (_("can't encode register '%s%s' with VEX/XOP/EVEX"),
 		    register_prefix, i.op[j].regs->reg_name);
diff --git a/opcodes/i386-reg.tbl b/opcodes/i386-reg.tbl
index d441f36..e314645 100644
--- a/opcodes/i386-reg.tbl
+++ b/opcodes/i386-reg.tbl
@@ -38,30 +38,30 @@ spl, Class=Reg|Byte, RegRex64, 4, Dw2Inval, Dw2Inval
 bpl, Class=Reg|Byte, RegRex64, 5, Dw2Inval, Dw2Inval
 sil, Class=Reg|Byte, RegRex64, 6, Dw2Inval, Dw2Inval
 dil, Class=Reg|Byte, RegRex64, 7, Dw2Inval, Dw2Inval
-r8b, Class=Reg|Byte, RegRex|RegRex64, 0, Dw2Inval, Dw2Inval
-r9b, Class=Reg|Byte, RegRex|RegRex64, 1, Dw2Inval, Dw2Inval
-r10b, Class=Reg|Byte, RegRex|RegRex64, 2, Dw2Inval, Dw2Inval
-r11b, Class=Reg|Byte, RegRex|RegRex64, 3, Dw2Inval, Dw2Inval
-r12b, Class=Reg|Byte, RegRex|RegRex64, 4, Dw2Inval, Dw2Inval
-r13b, Class=Reg|Byte, RegRex|RegRex64, 5, Dw2Inval, Dw2Inval
-r14b, Class=Reg|Byte, RegRex|RegRex64, 6, Dw2Inval, Dw2Inval
-r15b, Class=Reg|Byte, RegRex|RegRex64, 7, Dw2Inval, Dw2Inval
-r16b, Class=Reg|Byte, RegRex2|RegRex64, 0, Dw2Inval, Dw2Inval
-r17b, Class=Reg|Byte, RegRex2|RegRex64, 1, Dw2Inval, Dw2Inval
-r18b, Class=Reg|Byte, RegRex2|RegRex64, 2, Dw2Inval, Dw2Inval
-r19b, Class=Reg|Byte, RegRex2|RegRex64, 3, Dw2Inval, Dw2Inval
-r20b, Class=Reg|Byte, RegRex2|RegRex64, 4, Dw2Inval, Dw2Inval
-r21b, Class=Reg|Byte, RegRex2|RegRex64, 5, Dw2Inval, Dw2Inval
-r22b, Class=Reg|Byte, RegRex2|RegRex64, 6, Dw2Inval, Dw2Inval
-r23b, Class=Reg|Byte, RegRex2|RegRex64, 7, Dw2Inval, Dw2Inval
-r24b, Class=Reg|Byte, RegRex2|RegRex64|RegRex, 0, Dw2Inval, Dw2Inval
-r25b, Class=Reg|Byte, RegRex2|RegRex64|RegRex, 1, Dw2Inval, Dw2Inval
-r26b, Class=Reg|Byte, RegRex2|RegRex64|RegRex, 2, Dw2Inval, Dw2Inval
-r27b, Class=Reg|Byte, RegRex2|RegRex64|RegRex, 3, Dw2Inval, Dw2Inval
-r28b, Class=Reg|Byte, RegRex2|RegRex64|RegRex, 4, Dw2Inval, Dw2Inval
-r29b, Class=Reg|Byte, RegRex2|RegRex64|RegRex, 5, Dw2Inval, Dw2Inval
-r30b, Class=Reg|Byte, RegRex2|RegRex64|RegRex, 6, Dw2Inval, Dw2Inval
-r31b, Class=Reg|Byte, RegRex2|RegRex64|RegRex, 7, Dw2Inval, Dw2Inval
+r8b, Class=Reg|Byte, RegRex, 0, Dw2Inval, Dw2Inval
+r9b, Class=Reg|Byte, RegRex, 1, Dw2Inval, Dw2Inval
+r10b, Class=Reg|Byte, RegRex, 2, Dw2Inval, Dw2Inval
+r11b, Class=Reg|Byte, RegRex, 3, Dw2Inval, Dw2Inval
+r12b, Class=Reg|Byte, RegRex, 4, Dw2Inval, Dw2Inval
+r13b, Class=Reg|Byte, RegRex, 5, Dw2Inval, Dw2Inval
+r14b, Class=Reg|Byte, RegRex, 6, Dw2Inval, Dw2Inval
+r15b, Class=Reg|Byte, RegRex, 7, Dw2Inval, Dw2Inval
+r16b, Class=Reg|Byte, RegRex2, 0, Dw2Inval, Dw2Inval
+r17b, Class=Reg|Byte, RegRex2, 1, Dw2Inval, Dw2Inval
+r18b, Class=Reg|Byte, RegRex2, 2, Dw2Inval, Dw2Inval
+r19b, Class=Reg|Byte, RegRex2, 3, Dw2Inval, Dw2Inval
+r20b, Class=Reg|Byte, RegRex2, 4, Dw2Inval, Dw2Inval
+r21b, Class=Reg|Byte, RegRex2, 5, Dw2Inval, Dw2Inval
+r22b, Class=Reg|Byte, RegRex2, 6, Dw2Inval, Dw2Inval
+r23b, Class=Reg|Byte, RegRex2, 7, Dw2Inval, Dw2Inval
+r24b, Class=Reg|Byte, RegRex2|RegRex, 0, Dw2Inval, Dw2Inval
+r25b, Class=Reg|Byte, RegRex2|RegRex, 1, Dw2Inval, Dw2Inval
+r26b, Class=Reg|Byte, RegRex2|RegRex, 2, Dw2Inval, Dw2Inval
+r27b, Class=Reg|Byte, RegRex2|RegRex, 3, Dw2Inval, Dw2Inval
+r28b, Class=Reg|Byte, RegRex2|RegRex, 4, Dw2Inval, Dw2Inval
+r29b, Class=Reg|Byte, RegRex2|RegRex, 5, Dw2Inval, Dw2Inval
+r30b, Class=Reg|Byte, RegRex2|RegRex, 6, Dw2Inval, Dw2Inval
+r31b, Class=Reg|Byte, RegRex2|RegRex, 7, Dw2Inval, Dw2Inval
 // 16 bit regs
 ax, Class=Reg|Instance=Accum|Word, 0, 0, Dw2Inval, Dw2Inval
 cx, Class=Reg|Word, 0, 1, Dw2Inval, Dw2Inval
diff --git a/opcodes/i386-tbl.h b/opcodes/i386-tbl.h
index e42254b..88354c4 100644
--- a/opcodes/i386-tbl.h
+++ b/opcodes/i386-tbl.h
@@ -48674,99 +48674,99 @@ static const reg_entry i386_regtab[] =
   { "r8b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex|RegRex64, 0, { Dw2Inval, Dw2Inval } },
+    RegRex, 0, { Dw2Inval, Dw2Inval } },
   { "r9b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex|RegRex64, 1, { Dw2Inval, Dw2Inval } },
+    RegRex, 1, { Dw2Inval, Dw2Inval } },
   { "r10b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex|RegRex64, 2, { Dw2Inval, Dw2Inval } },
+    RegRex, 2, { Dw2Inval, Dw2Inval } },
   { "r11b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex|RegRex64, 3, { Dw2Inval, Dw2Inval } },
+    RegRex, 3, { Dw2Inval, Dw2Inval } },
   { "r12b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex|RegRex64, 4, { Dw2Inval, Dw2Inval } },
+    RegRex, 4, { Dw2Inval, Dw2Inval } },
   { "r13b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex|RegRex64, 5, { Dw2Inval, Dw2Inval } },
+    RegRex, 5, { Dw2Inval, Dw2Inval } },
   { "r14b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex|RegRex64, 6, { Dw2Inval, Dw2Inval } },
+    RegRex, 6, { Dw2Inval, Dw2Inval } },
   { "r15b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex|RegRex64, 7, { Dw2Inval, Dw2Inval } },
+    RegRex, 7, { Dw2Inval, Dw2Inval } },
   { "r16b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64, 0, { Dw2Inval, Dw2Inval } },
+    RegRex2, 0, { Dw2Inval, Dw2Inval } },
   { "r17b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64, 1, { Dw2Inval, Dw2Inval } },
+    RegRex2, 1, { Dw2Inval, Dw2Inval } },
   { "r18b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64, 2, { Dw2Inval, Dw2Inval } },
+    RegRex2, 2, { Dw2Inval, Dw2Inval } },
   { "r19b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64, 3, { Dw2Inval, Dw2Inval } },
+    RegRex2, 3, { Dw2Inval, Dw2Inval } },
   { "r20b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64, 4, { Dw2Inval, Dw2Inval } },
+    RegRex2, 4, { Dw2Inval, Dw2Inval } },
   { "r21b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64, 5, { Dw2Inval, Dw2Inval } },
+    RegRex2, 5, { Dw2Inval, Dw2Inval } },
   { "r22b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64, 6, { Dw2Inval, Dw2Inval } },
+    RegRex2, 6, { Dw2Inval, Dw2Inval } },
   { "r23b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64, 7, { Dw2Inval, Dw2Inval } },
+    RegRex2, 7, { Dw2Inval, Dw2Inval } },
   { "r24b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64|RegRex, 0, { Dw2Inval, Dw2Inval } },
+    RegRex2|RegRex, 0, { Dw2Inval, Dw2Inval } },
   { "r25b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64|RegRex, 1, { Dw2Inval, Dw2Inval } },
+    RegRex2|RegRex, 1, { Dw2Inval, Dw2Inval } },
   { "r26b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64|RegRex, 2, { Dw2Inval, Dw2Inval } },
+    RegRex2|RegRex, 2, { Dw2Inval, Dw2Inval } },
   { "r27b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64|RegRex, 3, { Dw2Inval, Dw2Inval } },
+    RegRex2|RegRex, 3, { Dw2Inval, Dw2Inval } },
   { "r28b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64|RegRex, 4, { Dw2Inval, Dw2Inval } },
+    RegRex2|RegRex, 4, { Dw2Inval, Dw2Inval } },
   { "r29b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64|RegRex, 5, { Dw2Inval, Dw2Inval } },
+    RegRex2|RegRex, 5, { Dw2Inval, Dw2Inval } },
   { "r30b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64|RegRex, 6, { Dw2Inval, Dw2Inval } },
+    RegRex2|RegRex, 6, { Dw2Inval, Dw2Inval } },
   { "r31b",
     { { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },
-    RegRex2|RegRex64|RegRex, 7, { Dw2Inval, Dw2Inval } },
+    RegRex2|RegRex, 7, { Dw2Inval, Dw2Inval } },
   { "ax",
     { { 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0 } },