x86-64: special case LEA when determining signedness of displacement

LEA behavior without a 64-bit destination is independent of address size
- in particular LEA with 32-bit addressing and 64-bit destination is the
same as LEA with 64-bit addressing and 32-bit destination. IOW checking
merely i.prefix[ADDR_PREFIX] is insufficient. This also means wrong
relocation types (R_X86_64_32S when R_X86_64_32 is needed) were used so
far in such cases.

Note that in one case in build_modrm_byte() the 64-bit check came too
early altogether, and hence gets dropped in favor of the one included in
the new helper. This is benign to non-64-bit code from all I can tell,
but the failure to clear disp16 could have been a latent problem.

2 files changed, 24 insertions, 8 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index aeacaa6..7fe0777 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,5 +1,12 @@
 2021-04-23  Jan Beulich  <jbeulich@suse.com>
 
+	* config/tc-i386.c (want_disp32): New.
+	(md_assemble): Use it.
+	(optimize_disp): Likewise.
+	(build_modrm_byte): Likewise.
+
+2021-04-23  Jan Beulich  <jbeulich@suse.com>
+
 	* config/tc-i386.c (i386_finalize_displacement): Move Disp32S
 	check ...
 	(md_assemble): ... here.
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 8098f50..dc1ee88 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -3555,6 +3555,16 @@ tc_i386_fix_adjustable (fixS *fixP ATTRIBUTE_UNUSED)
   return 1;
 }
 
+static INLINE bool
+want_disp32 (const insn_template *t)
+{
+  return flag_code != CODE_64BIT
+	 || i.prefix[ADDR_PREFIX]
+	 || (t->base_opcode == 0x8d
+	     && t->opcode_modifier.opcodespace == SPACE_BASE
+	     && !i.types[1].bitfield.qword);
+}
+
 static int
 intel_float_operand (const char *mnemonic)
 {
@@ -4740,7 +4750,7 @@ md_assemble (char *line)
   if (i.imm_operands)
     optimize_imm ();
 
-  if (i.disp_operands && flag_code == CODE_64BIT && !i.prefix[ADDR_PREFIX])
+  if (i.disp_operands && !want_disp32 (current_templates->start))
     {
       for (j = 0; j < i.operands; ++j)
 	{
@@ -5748,7 +5758,7 @@ optimize_disp (void)
 #ifdef BFD64
 	    else if (flag_code == CODE_64BIT)
 	      {
-		if (i.prefix[ADDR_PREFIX]
+		if (want_disp32 (current_templates->start)
 		    && fits_in_unsigned_long (op_disp))
 		  i.types[op].bitfield.disp32 = 1;
 
@@ -8108,7 +8118,7 @@ build_modrm_byte (void)
 		  i.types[op].bitfield.disp8 = 0;
 		  i.types[op].bitfield.disp16 = 0;
 		  i.types[op].bitfield.disp64 = 0;
-		  if (flag_code != CODE_64BIT || i.prefix[ADDR_PREFIX])
+		  if (want_disp32 (&i.tm))
 		    {
 		      /* Must be 32 bit */
 		      i.types[op].bitfield.disp32 = 1;
@@ -8159,7 +8169,7 @@ build_modrm_byte (void)
 		      i.rm.regmem = ESCAPE_TO_TWO_BYTE_ADDRESSING;
 		      i.sib.base = NO_BASE_REGISTER;
 		      i.sib.index = NO_INDEX_REGISTER;
-		      newdisp = (!i.prefix[ADDR_PREFIX] ? disp32s : disp32);
+		      newdisp = (want_disp32(&i.tm) ? disp32 : disp32s);
 		    }
 		  else if ((flag_code == CODE_16BIT)
 			   ^ (i.prefix[ADDR_PREFIX] != 0))
@@ -8188,7 +8198,7 @@ build_modrm_byte (void)
 		  i.types[op].bitfield.disp8 = 0;
 		  i.types[op].bitfield.disp16 = 0;
 		  i.types[op].bitfield.disp64 = 0;
-		  if (flag_code != CODE_64BIT || i.prefix[ADDR_PREFIX])
+		  if (want_disp32 (&i.tm))
 		    {
 		      /* Must be 32 bit */
 		      i.types[op].bitfield.disp32 = 1;
@@ -8263,12 +8273,11 @@ build_modrm_byte (void)
 	    }
 	  else /* i.base_reg and 32/64 bit mode  */
 	    {
-	      if (flag_code == CODE_64BIT
-		  && operand_type_check (i.types[op], disp))
+	      if (operand_type_check (i.types[op], disp))
 		{
 		  i.types[op].bitfield.disp16 = 0;
 		  i.types[op].bitfield.disp64 = 0;
-		  if (i.prefix[ADDR_PREFIX] == 0)
+		  if (!want_disp32 (&i.tm))
 		    {
 		      i.types[op].bitfield.disp32 = 0;
 		      i.types[op].bitfield.disp32s = 1;