x86: honor signedness of PC-relative relocations

PR gas/27763

While the comment in output_jump() was basically correct prior to the
introduction of 64-bit mode, both that and the not-JMP-like behavior of
XBEGIN require adjustments: Branches with 32-bit displacement do not
wrap at 4G in 64-bit mode, and XBEGIN with 16-bit operand size doesn't
wrap at 64k. Similarly %rip-relative addressing doesn't wrap at 4G.

The new testcase points out that for PE/COFF object_64bit didn't get
set so far, preventing in particular the check at the end of
md_convert_frag() to take effect.

For Mach-O the new testcase fails (bogusly), in that only the first two
of the expected errors get raised. Since for Mach-O many testcases
already fail, and since an x86_64-darwin target can't even be configured
for, I didn't think I need to bother.

Note that there are further issues in this area, in particular for
branches with operand size overrides. Such branches, which truncate
%rip / %eip, can't be correctly expressed with ordinary PC-relative
relocations. It's not really clear what to do with them - perhaps the
best we can do is to carry through all associated relocations, leaving
it to the linker (or even loader) to decide (once the final address
layout is known). Same perhaps goes for relocations associated with
32-bit addressing in 64-bit mode.

11 files changed, 211 insertions, 18 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 64378d1..5d341e8 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,22 @@
+2021-04-28  Jan Beulich  <jbeulich@suse.com>
+            H.J. Lu  <hjl.tools@gmail.com>
+
+	PR gas/27763
+	* config/tc-i386.c (output_jump): Also mark 2-byte relocs as
+	signed for XBEGIN. Also mark 4-byte relocs as signed for 64-bit.
+	(output_disp): Also mark 4-byte relocs as signed for 64-bit.
+	(md_estimate_size_before_relax): Move local variable fixP. Set
+	it from fix_new() return values. Mark 4-byte relocs as signed
+	for 64-bit.
+	* testsuite/gas/i386/pcrel64.s, testsuite/gas/i386/pcrel64.l,
+	* testsuite/gas/i386/x86-64-rip-2.s,
+	* testsuite/gas/i386/x86-64-rip-2.d,
+	* testsuite/gas/i386/x86-64-rip-inval-1.s,
+	* testsuite/gas/i386/x86-64-rip-inval-1.l,
+	* testsuite/gas/i386/x86-64-rip-inval-2.s,
+	* testsuite/gas/i386/x86-64-rip-inval-2.l: New.
+	* testsuite/gas/i386/i386.exp: Run new tests.
+
 2021-04-27  H.J. Lu  <hongjiu.lu@intel.com>
 
 	* config/tc-i386.c (optimize_encoding): Add () to silence GCC 5.
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 88b601f..bd2da65 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -8929,11 +8929,26 @@ output_jump (void)
   fixP = fix_new_exp (frag_now, p - frag_now->fr_literal, size,
 		      i.op[0].disps, 1, jump_reloc);
 
-  /* All jumps handled here are signed, but don't use a signed limit
-     check for 32 and 16 bit jumps as we want to allow wrap around at
-     4G and 64k respectively.  */
-  if (size == 1)
-    fixP->fx_signed = 1;
+  /* All jumps handled here are signed, but don't unconditionally use a
+     signed limit check for 32 and 16 bit jumps as we want to allow wrap
+     around at 4G (outside of 64-bit mode) and 64k (except for XBEGIN)
+     respectively.  */
+  switch (size)
+    {
+    case 1:
+      fixP->fx_signed = 1;
+      break;
+
+    case 2:
+      if (i.tm.base_opcode == 0xc7f8)
+	fixP->fx_signed = 1;
+      break;
+
+    case 4:
+      if (flag_code == CODE_64BIT)
+	fixP->fx_signed = 1;
+      break;
+    }
 }
 
 static void
@@ -10022,6 +10037,11 @@ output_disp (fragS *insn_start_frag, offsetT insn_start_off)
 	      fixP = fix_new_exp (frag_now, p - frag_now->fr_literal,
 				  size, i.op[n].disps, pcrel,
 				  reloc_type);
+
+	      if (flag_code == CODE_64BIT && size == 4 && pcrel
+		  && !i.prefix[ADDR_PREFIX])
+		fixP->fx_signed = 1;
+
 	      /* Check for "call/jmp *mem", "mov mem, %reg",
 		 "test %reg, mem" and "binop mem, %reg" where binop
 		 is one of adc, add, and, cmp, or, sbb, sub, xor
@@ -12257,6 +12277,7 @@ md_estimate_size_before_relax (fragS *fragP, segT segment)
       enum bfd_reloc_code_real reloc_type;
       unsigned char *opcode;
       int old_fr_fix;
+      fixS *fixP = NULL;
 
       if (fragP->fr_var != NO_RELOC)
 	reloc_type = (enum bfd_reloc_code_real) fragP->fr_var;
@@ -12278,10 +12299,10 @@ md_estimate_size_before_relax (fragS *fragP, segT segment)
 	  /* Make jmp (0xeb) a (d)word displacement jump.  */
 	  opcode[0] = 0xe9;
 	  fragP->fr_fix += size;
-	  fix_new (fragP, old_fr_fix, size,
-		   fragP->fr_symbol,
-		   fragP->fr_offset, 1,
-		   reloc_type);
+	  fixP = fix_new (fragP, old_fr_fix, size,
+			  fragP->fr_symbol,
+			  fragP->fr_offset, 1,
+			  reloc_type);
 	  break;
 
 	case COND_JUMP86:
@@ -12308,8 +12329,6 @@ md_estimate_size_before_relax (fragS *fragP, segT segment)
 	case COND_JUMP:
 	  if (no_cond_jump_promotion && fragP->fr_var == NO_RELOC)
 	    {
-	      fixS *fixP;
-
 	      fragP->fr_fix += 1;
 	      fixP = fix_new (fragP, old_fr_fix, 1,
 			      fragP->fr_symbol,
@@ -12325,16 +12344,23 @@ md_estimate_size_before_relax (fragS *fragP, segT segment)
 	  opcode[0] = TWO_BYTE_OPCODE_ESCAPE;
 	  /* We've added an opcode byte.  */
 	  fragP->fr_fix += 1 + size;
-	  fix_new (fragP, old_fr_fix + 1, size,
-		   fragP->fr_symbol,
-		   fragP->fr_offset, 1,
-		   reloc_type);
+	  fixP = fix_new (fragP, old_fr_fix + 1, size,
+			  fragP->fr_symbol,
+			  fragP->fr_offset, 1,
+			  reloc_type);
 	  break;
 
 	default:
 	  BAD_CASE (fragP->fr_subtype);
 	  break;
 	}
+
+      /* All jumps handled here are signed, but don't unconditionally use a
+	 signed limit check for 32 and 16 bit jumps as we want to allow wrap
+	 around at 4G (outside of 64-bit mode) and 64k.  */
+      if (size == 4 && flag_code == CODE_64BIT)
+	fixP->fx_signed = 1;
+
       frag_wane (fragP);
       return fragP->fr_fix - old_fr_fix;
     }
@@ -13966,9 +13992,11 @@ i386_target_format (void)
 # if defined (TE_PE) || defined (TE_PEP)
     case bfd_target_coff_flavour:
       if (flag_code == CODE_64BIT)
-	return use_big_obj ? "pe-bigobj-x86-64" : "pe-x86-64";
-      else
-	return use_big_obj ? "pe-bigobj-i386" : "pe-i386";
+	{
+	  object_64bit = 1;
+	  return use_big_obj ? "pe-bigobj-x86-64" : "pe-x86-64";
+	}
+      return use_big_obj ? "pe-bigobj-i386" : "pe-i386";
 # elif defined (TE_GO32)
     case bfd_target_coff_flavour:
       return "coff-go32";
diff --git a/gas/testsuite/gas/i386/i386.exp b/gas/testsuite/gas/i386/i386.exp
index 0086645..1846c98 100644
--- a/gas/testsuite/gas/i386/i386.exp
+++ b/gas/testsuite/gas/i386/i386.exp
@@ -760,8 +760,12 @@ if [gas_64_check] then {
     } else {
       run_dump_test "x86-64-w64-pcrel"
     }
+    run_list_test "pcrel64" "-al"
     run_dump_test "x86-64-rip"
     run_dump_test "x86-64-rip-intel"
+    run_dump_test "x86-64-rip-2"
+    run_list_test "x86-64-rip-inval-1" "-al"
+    run_list_test "x86-64-rip-inval-2" "-al"
     run_dump_test "x86-64-stack"
     run_dump_test "x86-64-stack-intel"
     run_dump_test "x86-64-stack-suffix"
diff --git a/gas/testsuite/gas/i386/pcrel64.l b/gas/testsuite/gas/i386/pcrel64.l
new file mode 100644
index 0000000..7c2fc64
--- /dev/null
+++ b/gas/testsuite/gas/i386/pcrel64.l
@@ -0,0 +1,54 @@
+.*: Assembler messages:
+.*:16: Error: .*
+.*:17: Error: .*
+.*:13: Error: .*
+.*:15: Error: .*
+.*:18: Error: .*
+.*:19: Error: .*
+.*:20: Error: .*
+GAS LISTING .*
+
+
+[ 	]*[0-9]+[ 	]+\.text
+[ 	]*[0-9]+[ 	]+\.code64
+[ 	]*[0-9]+[ 	]+pcrel:
+[ 	]*[0-9]+ \?\?\?\? E8..8000[ 	]+call	target
+[ 	]*[0-9]+[ 	]+00
+[ 	]*[0-9]+ \?\?\?\? E9..8000[ 	]+jmp	target
+[ 	]*[0-9]+[ 	]+00
+[ 	]*[0-9]+ \?\?\?\? 0F84..80[ 	]+jz	target
+[ 	]*[0-9]+[ 	]+0000
+[ 	]*[0-9]+ \?\?\?\? C7F8..80[ 	]+xbegin	target
+[ 	]*[0-9]+[ 	]+0000
+[ 	]*[0-9]+ \?\?\?\? 8B05..80[ 	]+mov	target\(%rip\), %eax
+[ 	]*[0-9]+[ 	]+0000
+[ 	]*[0-9]+ \?\?\?\? 678B05..[ 	]+mov	target\(%eip\), %eax
+[ 	]*[0-9]+[ 	]+800000
+[ 	]*[0-9]+ \?\?\?\? 48C7C0..[ 	]+mov	\$target-., %rax
+[ 	]*[0-9]+[ 	]+800000
+[ 	]*[0-9]+ \?\?\?\? B8..8000[ 	]+mov	\$target-., %eax
+[ 	]*[0-9]+[ 	]+00
+[ 	]*[0-9]+[ 	]*
+[ 	]*[0-9]+ \?\?\?\? 66C7F8..[ 	]+data16 xbegin target
+[ 	]*[0-9]+[ 	]+80
+[ 	]*[0-9]+[ 	]*
+[ 	]*[0-9]+ \?\?\?\? E8...000[ 	]+call	target\+0x7ffff000
+[ 	]*[0-9]+[ 	]+80
+[ 	]*[0-9]+ \?\?\?\? E9000000[ 	]+jmp	target\+0x7ffff000
+[ 	]*[0-9]+[ 	]+00
+[ 	]*[0-9]+ \?\?\?\? 0F840000[ 	]+jz	target\+0x7ffff000
+[ 	]*[0-9]+[ 	]+0000
+[ 	]*[0-9]+ \?\?\?\? C7F8...0[ 	]+xbegin	target\+0x7ffff000
+[ 	]*[0-9]+[ 	]+0080
+[ 	]*[0-9]+ \?\?\?\? 8B05...0[ 	]+mov	target\+0x7ffff000\(%rip\), %eax
+[ 	]*[0-9]+[ 	]+0080
+[ 	]*[0-9]+ \?\?\?\? 48C7C0..[ 	]+mov	\$target\+0x7ffff000-., %rax
+[ 	]*[0-9]+[ 	]+.00080
+[ 	]*[0-9]+[ 	]*
+[ 	]*[0-9]+ \?\?\?\? 678B05..[ 	]+mov	target\+0x7ffff000\(%eip\), %eax
+[ 	]*[0-9]+[ 	]+.00080
+[ 	]*[0-9]+ \?\?\?\? B8...000[ 	]+mov	\$target\+0x7ffff000-., %eax
+[ 	]*[0-9]+[ 	]+80
+[ 	]*[0-9]+[ 	]*
+[ 	]*[0-9]+ \?\?\?\? CCCCCCCC[ 	]+\.fill 0x8000, 1, 0xcc
+#pass
diff --git a/gas/testsuite/gas/i386/pcrel64.s b/gas/testsuite/gas/i386/pcrel64.s
new file mode 100644
index 0000000..831ee40
--- /dev/null
+++ b/gas/testsuite/gas/i386/pcrel64.s
@@ -0,0 +1,27 @@
+	.text
+	.code64
+pcrel:
+	call	target
+	jmp	target
+	jz	target
+	xbegin	target
+	mov	target(%rip), %eax
+	mov	target(%eip), %eax
+	mov	$target-., %rax
+	mov	$target-., %eax
+
+	data16 xbegin target
+
+	call	target+0x7ffff000
+	jmp	target+0x7ffff000
+	jz	target+0x7ffff000
+	xbegin	target+0x7ffff000
+	mov	target+0x7ffff000(%rip), %eax
+	mov	$target+0x7ffff000-., %rax
+
+	mov	target+0x7ffff000(%eip), %eax
+	mov	$target+0x7ffff000-., %eax
+
+	.fill 0x8000, 1, 0xcc
+target:
+	ret
diff --git a/gas/testsuite/gas/i386/x86-64-rip-2.d b/gas/testsuite/gas/i386/x86-64-rip-2.d
new file mode 100644
index 0000000..bf9ca39
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-rip-2.d
@@ -0,0 +1,21 @@
+#as: -J
+#objdump: -drw --syms
+#name: x86-64 rip addressing 2
+
+.*: +file format .*
+
+SYMBOL TABLE:
+0000000000000000 l       .text	0000000000000000 _start
+0000000080000006 l       .text	0000000000000000 test1
+ffffffff8000000e l       .text	0000000000000000 test2
+00000000f000000e l       .text	0000000000000000 test3
+ffffffff1000000e l       .text	0000000000000000 test4
+
+
+
+Disassembly of section .text:
+
+0000000000000000 <_start>:
+ +0:	48 8b 05 ff ff ff 7f 	mov    0x7fffffff\(%rip\),%rax        # 80000006 <test1>
+ +7:	48 8b 05 00 00 00 80 	mov    -0x80000000\(%rip\),%rax        # ffffffff8000000e <test2>
+#pass
diff --git a/gas/testsuite/gas/i386/x86-64-rip-2.s b/gas/testsuite/gas/i386/x86-64-rip-2.s
new file mode 100644
index 0000000..5ce80e8
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-rip-2.s
@@ -0,0 +1,10 @@
+	.text
+_start:
+	movq test1(%rip), %rax
+	.set test1, . + 0x7fffffff
+
+	movq test2(%rip), %rax
+	.set test2, . - 0x80000000
+
+	.set test3, . + 0xf0000000
+	.set test4, . - 0xf0000000
diff --git a/gas/testsuite/gas/i386/x86-64-rip-inval-1.l b/gas/testsuite/gas/i386/x86-64-rip-inval-1.l
new file mode 100644
index 0000000..d40099e
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-rip-inval-1.l
@@ -0,0 +1,11 @@
+.*: Assembler messages:
+.*:3: Error:.* (0x)?ffffffff7fffffff .*
+GAS LISTING .*
+
+
+[ 	]*1[ 	]+\.text
+[ 	]*2[ 	]+_start:
+[ 	]*3[ 	]+\?\?\?\? 488B05FF 		movq test1\(%rip\), %rax
+[ 	]*3[ 	]+FFFF7F
+[ 	]*4[ 	]+\.set test1, \. - 0x80000001
+#pass
diff --git a/gas/testsuite/gas/i386/x86-64-rip-inval-1.s b/gas/testsuite/gas/i386/x86-64-rip-inval-1.s
new file mode 100644
index 0000000..a0e783e
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-rip-inval-1.s
@@ -0,0 +1,4 @@
+	.text
+_start:
+	movq test1(%rip), %rax
+	.set test1, . - 0x80000001
diff --git a/gas/testsuite/gas/i386/x86-64-rip-inval-2.l b/gas/testsuite/gas/i386/x86-64-rip-inval-2.l
new file mode 100644
index 0000000..e724393
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-rip-inval-2.l
@@ -0,0 +1,11 @@
+.*: Assembler messages:
+.*:3: Error:.* (0x)?0*80000000 .*
+GAS LISTING .*
+
+
+[ 	]*1[ 	]+\.text
+[ 	]*2[ 	]+_start:
+[ 	]*3[ 	]+\?\?\?\? 488B0500 		movq test1\(%rip\), %rax
+[ 	]*3[ 	]+000080
+[ 	]*4[ 	]+\.set test1, \. \+ 0x80000000
+#pass
diff --git a/gas/testsuite/gas/i386/x86-64-rip-inval-2.s b/gas/testsuite/gas/i386/x86-64-rip-inval-2.s
new file mode 100644
index 0000000..9bdc703
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-rip-inval-2.s
@@ -0,0 +1,4 @@
+	.text
+_start:
+	movq test1(%rip), %rax
+	.set test1, . + 0x80000000