x86-64/Intel: issue diagnostic for out of range displacement

... rather than silently dropping it altogether.
i386_finalize_displacement() expects baseindex to already be set, so
the respective statement needs to be moved up. This then also allows a
subsequent conditional to be simplified.

For this to not regress on 32-bit addressing, break out address size
guessing from i386_index_check(), invoking the new function earlier so
that i386_finalize_displacement() has i.prefix[ADDR_PREFIX] available.
i386_addressing_mode () in turn needs i.base_reg / i.index_reg set
earlier.

2 files changed, 42 insertions, 29 deletions

diff --git a/gas/config/tc-i386-intel.c b/gas/config/tc-i386-intel.c
index 9e8135e..36ae818 100644
--- a/gas/config/tc-i386-intel.c
+++ b/gas/config/tc-i386-intel.c
@@ -876,18 +876,41 @@ i386_intel_operand (char *operand_string, int got_a_float)
 	  return 0;
 	}
 
+      /* Swap base and index in 16-bit memory operands like
+	 [si+bx]. Since i386_index_check is also used in AT&T
+	 mode we have to do this here.  */
+      if (intel_state.base
+	  && intel_state.index
+	  && intel_state.base->reg_type.bitfield.reg16
+	  && intel_state.index->reg_type.bitfield.reg16
+	  && intel_state.base->reg_num >= 6
+	  && intel_state.index->reg_num < 6)
+	{
+	  i.base_reg = intel_state.index;
+	  i.index_reg = intel_state.base;
+	}
+      else
+	{
+	  i.base_reg = intel_state.base;
+	  i.index_reg = intel_state.index;
+	}
+
+      if (i.base_reg || i.index_reg)
+	i.types[this_operand].bitfield.baseindex = 1;
+
       expP = &disp_expressions[i.disp_operands];
       memcpy (expP, &exp, sizeof(exp));
       resolve_expression (expP);
 
       if (expP->X_op != O_constant
 	  || expP->X_add_number
-	  || (!intel_state.base
-	      && !intel_state.index))
+	  || !i.types[this_operand].bitfield.baseindex)
 	{
 	  i.op[this_operand].disps = expP;
 	  i.disp_operands++;
 
+	  i386_addressing_mode ();
+
 	  if (flag_code == CODE_64BIT)
 	    {
 	      i.types[this_operand].bitfield.disp32 = 1;
@@ -927,9 +950,6 @@ i386_intel_operand (char *operand_string, int got_a_float)
 	    return 0;
 	}
 
-      if (intel_state.base || intel_state.index)
-	i.types[this_operand].bitfield.baseindex = 1;
-
       if (intel_state.seg)
 	{
 	  expP = symbol_get_value_expression (intel_state.seg);
@@ -956,25 +976,6 @@ i386_intel_operand (char *operand_string, int got_a_float)
 	    }
 	}
 
-      /* Swap base and index in 16-bit memory operands like
-	 [si+bx]. Since i386_index_check is also used in AT&T
-	 mode we have to do that here.  */
-      if (intel_state.base
-	  && intel_state.index
-	  && intel_state.base->reg_type.bitfield.reg16
-	  && intel_state.index->reg_type.bitfield.reg16
-	  && intel_state.base->reg_num >= 6
-	  && intel_state.index->reg_num < 6)
-	{
-	  i.base_reg = intel_state.index;
-	  i.index_reg = intel_state.base;
-	}
-      else
-	{
-	  i.base_reg = intel_state.base;
-	  i.index_reg = intel_state.index;
-	}
-
       if (!i386_index_check (operand_string))
 	return 0;
 
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index dcc70c8..4161c8c 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -8633,13 +8633,13 @@ i386_finalize_displacement (segT exp_seg ATTRIBUTE_UNUSED, expressionS *exp,
   return ret;
 }
 
-/* Make sure the memory operand we've been dealt is valid.
-   Return 1 on success, 0 on a failure.  */
+/* Return the active addressing mode, taking address override and
+   registers forming the address into consideration.  Update the
+   address override prefix if necessary.  */
 
-static int
-i386_index_check (const char *operand_string)
+static enum flag_code
+i386_addressing_mode (void)
 {
-  const char *kind = "base/index";
   enum flag_code addr_mode;
 
   if (i.prefix[ADDR_PREFIX])
@@ -8688,6 +8688,18 @@ i386_index_check (const char *operand_string)
 #endif
     }
 
+  return addr_mode;
+}
+
+/* Make sure the memory operand we've been dealt is valid.
+   Return 1 on success, 0 on a failure.  */
+
+static int
+i386_index_check (const char *operand_string)
+{
+  const char *kind = "base/index";
+  enum flag_code addr_mode = i386_addressing_mode ();
+
   if (current_templates->start->opcode_modifier.isstring
       && !current_templates->start->opcode_modifier.immext
       && (current_templates->end[-1].opcode_modifier.isstring