x86: consolidate Disp<NN> handling a little

In memory operand addressing, which forms of displacement are permitted
besides Disp8 is pretty clearly limited
- outside of 64-bit mode, Disp16 or Disp32 only, depending on address
  size (MPX being special in not allowing Disp16),
- in 64-bit mode, Disp32s or Disp64 without address size override, and
  solely Disp32 with one.
Adjust assembler and i386-gen to match this, observing that templates
already get adjusted before trying to match them against input depending
on the presence of an address size prefix.

This adjustment logic gets extended to all cases, as certain DispNN
values should also be dropped when there's no such prefix. In fact
behavior of the assembler, perhaps besides the exact diagnostics wording,
should not differ between there being templates applicable to 64-bit and
non-64-bit at the same time, or there being fully separate sets of
templates, with their DispNN settings already reduced accordingly.

This adjustment logic further gets guarded such that there wouldn't be
and Disp<N> conversion based on address size prefix when this prefix
doesn't control the width of the displacement (on branches other than
absolute ones).

These adjustments then also allow folding two MOV templates, which had
been split between 64-bit and non-64-bits variants so far.

Once in this area also
- drop the bogus DispNN from JumpByte templates, leaving just the
  correct Disp8 there (compensated by i386_finalize_displacement()
  now setting Disp8 on their operands),
- add the missing Disp32S to XBEGIN.

Note that the changes make it necessary to temporarily mark a test as
XFAIL; this will get taken care of by a subsequent patch. The failing
parts are entirely bogus and will get replaced.

4 files changed, 65 insertions, 46 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 875bcf8..9a51605 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,16 @@
+2019-12-27  Jan Beulich  <jbeulich@suse.com>
+
+	* config/tc-i386.c (i386_addressing_mode): Declare.
+	(match_template): Don't transform displacement width flags for
+	non-indirect branches. Re-write transformation logic.
+	(i386_displacement): Also check BaseIndex when deciding whether
+	an operand belongs to a direct branch. Restrict which DispNN get
+	set.
+	(i386_finalize_displacement): Set Disp8 for JumpByte templates.
+	* config/tc-i386-intel.c (i386_intel_operand): Don't set Disp32
+	for 64-bit addressing.
+	* testsuite/gas/i386/i386.exp: XFail x86-64-branch-3.
+
 2019-12-17  Alan Modra  <amodra@gmail.com>
 
 	* doc/as.texi: Remove mention of tic80.
diff --git a/gas/config/tc-i386-intel.c b/gas/config/tc-i386-intel.c
index bd88592..15993d0 100644
--- a/gas/config/tc-i386-intel.c
+++ b/gas/config/tc-i386-intel.c
@@ -939,12 +939,13 @@ i386_intel_operand (char *operand_string, int got_a_float)
 
 	  if (flag_code == CODE_64BIT)
 	    {
-	      i.types[this_operand].bitfield.disp32 = 1;
 	      if (!i.prefix[ADDR_PREFIX])
 		{
 		  i.types[this_operand].bitfield.disp64 = 1;
 		  i.types[this_operand].bitfield.disp32s = 1;
 		}
+	      else
+		i.types[this_operand].bitfield.disp32 = 1;
 	    }
 	  else if (!i.prefix[ADDR_PREFIX] ^ (flag_code == CODE_16BIT))
 	    i.types[this_operand].bitfield.disp32 = 1;
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 0dc44a8..155e636 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -182,6 +182,7 @@ static char *parse_insn (char *, char *);
 static char *parse_operands (char *, const char *);
 static void swap_operands (void);
 static void swap_2_operands (int, int);
+static enum flag_code i386_addressing_mode (void);
 static void optimize_imm (void);
 static void optimize_disp (void);
 static const insn_template *match_template (char);
@@ -5883,51 +5884,50 @@ match_template (char mnem_suffix)
 	    break;
 	}
 
-      /* Address size prefix will turn Disp64/Disp32/Disp16 operand
-	 into Disp32/Disp16/Disp32 operand.  */
-      if (i.prefix[ADDR_PREFIX] != 0)
-	  {
-	    /* There should be only one Disp operand.  */
-	    switch (flag_code)
-	    {
-	    case CODE_16BIT:
-	      for (j = 0; j < MAX_OPERANDS; j++)
-		{
-		  if (operand_types[j].bitfield.disp16)
-		    {
-		      addr_prefix_disp = j;
-		      operand_types[j].bitfield.disp32 = 1;
-		      operand_types[j].bitfield.disp16 = 0;
-		      break;
-		    }
-		}
+      if (!t->opcode_modifier.jump
+	  || t->opcode_modifier.jump == JUMP_ABSOLUTE)
+	{
+	  /* There should be only one Disp operand.  */
+	  for (j = 0; j < MAX_OPERANDS; j++)
+	    if (operand_type_check (operand_types[j], disp))
 	      break;
-	    case CODE_32BIT:
-	      for (j = 0; j < MAX_OPERANDS; j++)
+	  if (j < MAX_OPERANDS)
+	    {
+	      bfd_boolean override = (i.prefix[ADDR_PREFIX] != 0);
+
+	      addr_prefix_disp = j;
+
+	      /* Address size prefix will turn Disp64/Disp32S/Disp32/Disp16
+		 operand into Disp32/Disp32/Disp16/Disp32 operand.  */
+	      switch (flag_code)
 		{
-		  if (operand_types[j].bitfield.disp32)
+		case CODE_16BIT:
+		  override = !override;
+		  /* Fall through.  */
+		case CODE_32BIT:
+		  if (operand_types[j].bitfield.disp32
+		      && operand_types[j].bitfield.disp16)
 		    {
-		      addr_prefix_disp = j;
-		      operand_types[j].bitfield.disp32 = 0;
-		      operand_types[j].bitfield.disp16 = 1;
-		      break;
+		      operand_types[j].bitfield.disp16 = override;
+		      operand_types[j].bitfield.disp32 = !override;
 		    }
-		}
-	      break;
-	    case CODE_64BIT:
-	      for (j = 0; j < MAX_OPERANDS; j++)
-		{
-		  if (operand_types[j].bitfield.disp64)
+		  operand_types[j].bitfield.disp32s = 0;
+		  operand_types[j].bitfield.disp64 = 0;
+		  break;
+
+		case CODE_64BIT:
+		  if (operand_types[j].bitfield.disp32s
+		      || operand_types[j].bitfield.disp64)
 		    {
-		      addr_prefix_disp = j;
-		      operand_types[j].bitfield.disp64 = 0;
-		      operand_types[j].bitfield.disp32 = 1;
-		      break;
+		      operand_types[j].bitfield.disp64 &= !override;
+		      operand_types[j].bitfield.disp32s &= !override;
+		      operand_types[j].bitfield.disp32 = override;
 		    }
+		  operand_types[j].bitfield.disp16 = 0;
+		  break;
 		}
-	      break;
 	    }
-	  }
+	}
 
       /* Force 0x8b encoding for "mov foo@GOT, %eax".  */
       if (i.reloc[0] == BFD_RELOC_386_GOT32 && t->base_opcode == 0xa0)
@@ -9937,10 +9937,11 @@ i386_displacement (char *disp_start, char *disp_end)
 
   operand_type_set (&bigdisp, 0);
   if (i.jumpabsolute
+      || i.types[this_operand].bitfield.baseindex
       || (current_templates->start->opcode_modifier.jump != JUMP
 	  && current_templates->start->opcode_modifier.jump != JUMP_DWORD))
     {
-      bigdisp.bitfield.disp32 = 1;
+      i386_addressing_mode ();
       override = (i.prefix[ADDR_PREFIX] != 0);
       if (flag_code == CODE_64BIT)
 	{
@@ -9949,12 +9950,13 @@ i386_displacement (char *disp_start, char *disp_end)
 	      bigdisp.bitfield.disp32s = 1;
 	      bigdisp.bitfield.disp64 = 1;
 	    }
+	  else
+	    bigdisp.bitfield.disp32 = 1;
 	}
       else if ((flag_code == CODE_16BIT) ^ override)
-	{
-	  bigdisp.bitfield.disp32 = 0;
 	  bigdisp.bitfield.disp16 = 1;
-	}
+      else
+	  bigdisp.bitfield.disp32 = 1;
     }
   else
     {
@@ -9966,10 +9968,7 @@ i386_displacement (char *disp_start, char *disp_end)
 	  if (override || i.suffix == WORD_MNEM_SUFFIX)
 	    bigdisp.bitfield.disp16 = 1;
 	  else
-	    {
-	      bigdisp.bitfield.disp32 = 1;
-	      bigdisp.bitfield.disp32s = 1;
-	    }
+	    bigdisp.bitfield.disp32s = 1;
 	}
       else
 	{
@@ -10142,6 +10141,11 @@ i386_finalize_displacement (segT exp_seg ATTRIBUTE_UNUSED, expressionS *exp,
     }
 #endif
 
+  if (current_templates->start->opcode_modifier.jump == JUMP_BYTE
+      /* Constants get taken care of by optimize_disp().  */
+      && exp->X_op != O_constant)
+    i.types[this_operand].bitfield.disp8 = 1;
+
   /* Check if this is a displacement only operand.  */
   bigdisp = i.types[this_operand];
   bigdisp.bitfield.disp8 = 0;
diff --git a/gas/testsuite/gas/i386/i386.exp b/gas/testsuite/gas/i386/i386.exp
index c31ffab..8caf8ff 100644
--- a/gas/testsuite/gas/i386/i386.exp
+++ b/gas/testsuite/gas/i386/i386.exp
@@ -1120,6 +1120,7 @@ if [expr ([istarget "i*86-*-*"] || [istarget "x86_64-*-*"]) && [gas_64_check]] t
 
 	run_dump_test "x86-64-jump"
 	run_dump_test "x86-64-branch-2"
+	setup_xfail "*-*-*"
 	run_list_test "x86-64-branch-3" "-al -mintel64"
 	run_list_test "x86-64-branch-4" "-al -mintel64"