x86: Rewrite NOP generation for fill and alignment

Rewrite NOP generation for fill and code alignment by:

1. Add a 11-byte NOP with another 0x66 prefix.
2. Remove the multi-byte NOP entries which consist of 2 instructions.
3. Select proper NOPs based on ISA and processor tuning.
4. Generate multiple NOPs with the longer NOPs first followed by the
shorter NOP.
5. Use jump for larger NOP padding:
   a. > 8 bytes (2 NOPs) in 16-bit mode.
   b. > 14 bytes (2 NOPs) for older processors.
   c. > 77 bytes (7 NOPs) for newer processors.
6. Update MAX_MEM_FOR_RS_ALIGN_CODE to 4095.

	PR gas/22874
	* config/tc-i386.c (f32_5): Removed.
	(f32_8): Likewise.
	(f32_9): Likewise.
	(f32_10): Likewise.
	(f32_11): Likewise.
	(f32_12): Likewise.
	(f32_13): Likewise.
	(f32_14): Likewise.
	(f16_5): Likewise.
	(f16_6): Likewise.
	(f16_7): Likewise.
	(f16_8): Likewise.
	(jump_31): Likewise.
	(alt64_11): Likewise.
	(alt64_patt): Likewise.
	(jump_disp8): New.
	(jump32_disp32): Likewise.
	(jump16_disp32): Likewise.
	(alt_11): Likewise.
	(f32_patt): Updated.
	(f16_patt): Likewise.
	(alt_patt): Add alt_11.
	(i386_align_code): Merged with ...
	(i386_generate_nops): This.  Rewritten.
	(fits_in_imm7): Moved before i386_generate_nops.
	(fits_in_imm31): Likewise.
	* config/tc-i386.h (MAX_MEM_FOR_RS_ALIGN_CODE): Updated to
	4095.
	(i386_align_code): Removed.
	(HANDLE_ALIGN): Rewritten with i386_generate_nops.
	* doc/as.texinfo: Update limits of control byte for x86 .nops
	directive.
	* testsuite/gas/i386/i386.exp: Run nops-7 and x86-64-nops-7.
	* gas/testsuite/gas/i386/noavx-3.l: Updated.
	* gas/testsuite/gas/i386/nop-1.d: Likewise.
	* gas/testsuite/gas/i386/nop-1.s: Likewise.
	* gas/testsuite/gas/i386/nop-2.d: Likewise.
	* gas/testsuite/gas/i386/nop-2.s: Likewise.
	* gas/testsuite/gas/i386/nop-3.d: Likewise.
	* gas/testsuite/gas/i386/nop-4.d: Likewise.
	* gas/testsuite/gas/i386/nop-5.d: Likewise.
	* gas/testsuite/gas/i386/nop-5.s: Likewise.
	* gas/testsuite/gas/i386/nop-6.d: Likewise.
	* gas/testsuite/gas/i386/nop-bad-1.l: Likewise.
	* gas/testsuite/gas/i386/nops-1-core2.d: Likewise.
	* gas/testsuite/gas/i386/nops-1-i386-i686.d: Likewise.
	* gas/testsuite/gas/i386/nops-1-i386.d: Likewise.
	* gas/testsuite/gas/i386/nops-1-i686.d: Likewise.
	* gas/testsuite/gas/i386/nops-1-k8.d: Likewise.
	* gas/testsuite/gas/i386/nops-1.d: Likewise.
	* gas/testsuite/gas/i386/nops-2-core2.d: Likewise.
	* gas/testsuite/gas/i386/nops-2-i386.d: Likewise.
	* gas/testsuite/gas/i386/nops-2.d: Likewise.
	* gas/testsuite/gas/i386/nops-3-i386.d: Likewise.
	* gas/testsuite/gas/i386/nops-3-i686.d: Likewise.
	* gas/testsuite/gas/i386/nops-3.d: Likewise.
	* gas/testsuite/gas/i386/nops-4-i386.d: Likewise.
	* gas/testsuite/gas/i386/nops-4-i686.d: Likewise.
	* gas/testsuite/gas/i386/nops-4.d: Likewise.
	* gas/testsuite/gas/i386/nops-4a-i686.d: Likewise.
	* gas/testsuite/gas/i386/nops-5-i686.d: Likewise.
	* gas/testsuite/gas/i386/nops-5.d: Likewise.
	* gas/testsuite/gas/i386/nops-6.d: Likewise.
	* gas/testsuite/gas/i386/nops16-1.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nop-1.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nop-2.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nop-5.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-1-core2.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-1-g64.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-1-k8.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-1-pentium.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-1.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-2.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-3.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-4-core2.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-4-k8.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-4.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-5-k8.d: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-5.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-1-core2.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-1-k8.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-1-pentium.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-1.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-2.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-3.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-4-core2.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-4-k8.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-4.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-5-k8.d: Likewise.
	* gas/testsuite/gas/i386/ilp32/x86-64-nops-5.d: Likewise.
	* gas/testsuite/gas/i386/nops-7.d: New file.
	* gas/testsuite/gas/i386/nops-7.s: Likewise.
	* gas/testsuite/gas/i386/x86-64-nops-7.d: Likewise.

2 files changed, 147 insertions, 154 deletions

diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 4174d19..a10a36c 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -1179,63 +1179,27 @@ static const unsigned char f32_3[] =
   {0x8d,0x76,0x00};			/* leal 0(%esi),%esi	*/
 static const unsigned char f32_4[] =
   {0x8d,0x74,0x26,0x00};		/* leal 0(%esi,1),%esi	*/
-static const unsigned char f32_5[] =
-  {0x90,				/* nop			*/
-   0x8d,0x74,0x26,0x00};		/* leal 0(%esi,1),%esi	*/
 static const unsigned char f32_6[] =
   {0x8d,0xb6,0x00,0x00,0x00,0x00};	/* leal 0L(%esi),%esi	*/
 static const unsigned char f32_7[] =
   {0x8d,0xb4,0x26,0x00,0x00,0x00,0x00};	/* leal 0L(%esi,1),%esi */
-static const unsigned char f32_8[] =
-  {0x90,				/* nop			*/
-   0x8d,0xb4,0x26,0x00,0x00,0x00,0x00};	/* leal 0L(%esi,1),%esi */
-static const unsigned char f32_9[] =
-  {0x89,0xf6,				/* movl %esi,%esi	*/
-   0x8d,0xbc,0x27,0x00,0x00,0x00,0x00};	/* leal 0L(%edi,1),%edi */
-static const unsigned char f32_10[] =
-  {0x8d,0x76,0x00,			/* leal 0(%esi),%esi	*/
-   0x8d,0xbc,0x27,0x00,0x00,0x00,0x00};	/* leal 0L(%edi,1),%edi */
-static const unsigned char f32_11[] =
-  {0x8d,0x74,0x26,0x00,			/* leal 0(%esi,1),%esi	*/
-   0x8d,0xbc,0x27,0x00,0x00,0x00,0x00};	/* leal 0L(%edi,1),%edi */
-static const unsigned char f32_12[] =
-  {0x8d,0xb6,0x00,0x00,0x00,0x00,	/* leal 0L(%esi),%esi	*/
-   0x8d,0xbf,0x00,0x00,0x00,0x00};	/* leal 0L(%edi),%edi	*/
-static const unsigned char f32_13[] =
-  {0x8d,0xb6,0x00,0x00,0x00,0x00,	/* leal 0L(%esi),%esi	*/
-   0x8d,0xbc,0x27,0x00,0x00,0x00,0x00};	/* leal 0L(%edi,1),%edi */
-static const unsigned char f32_14[] =
-  {0x8d,0xb4,0x26,0x00,0x00,0x00,0x00,	/* leal 0L(%esi,1),%esi */
-   0x8d,0xbc,0x27,0x00,0x00,0x00,0x00};	/* leal 0L(%edi,1),%edi */
 static const unsigned char f16_3[] =
-  {0x8d,0x74,0x00};			/* lea 0(%esi),%esi	*/
+  {0x8d,0x74,0x00};			/* lea 0(%si),%si	*/
 static const unsigned char f16_4[] =
-  {0x8d,0xb4,0x00,0x00};		/* lea 0w(%si),%si	*/
-static const unsigned char f16_5[] =
-  {0x90,				/* nop			*/
-   0x8d,0xb4,0x00,0x00};		/* lea 0w(%si),%si	*/
-static const unsigned char f16_6[] =
-  {0x89,0xf6,				/* mov %si,%si		*/
-   0x8d,0xbd,0x00,0x00};		/* lea 0w(%di),%di	*/
-static const unsigned char f16_7[] =
-  {0x8d,0x74,0x00,			/* lea 0(%si),%si	*/
-   0x8d,0xbd,0x00,0x00};		/* lea 0w(%di),%di	*/
-static const unsigned char f16_8[] =
-  {0x8d,0xb4,0x00,0x00,			/* lea 0w(%si),%si	*/
-   0x8d,0xbd,0x00,0x00};		/* lea 0w(%di),%di	*/
-static const unsigned char jump_31[] =
-  {0xeb,0x1d,0x90,0x90,0x90,0x90,0x90,	/* jmp .+31; lotsa nops	*/
-   0x90,0x90,0x90,0x90,0x90,0x90,0x90,0x90,
-   0x90,0x90,0x90,0x90,0x90,0x90,0x90,0x90,
-   0x90,0x90,0x90,0x90,0x90,0x90,0x90,0x90};
+  {0x8d,0xb4,0x00,0x00};		/* lea 0W(%si),%si	*/
+static const unsigned char jump_disp8[] =
+  {0xeb};				/* jmp disp8	       */
+static const unsigned char jump32_disp32[] =
+  {0xe9};				/* jmp disp32	       */
+static const unsigned char jump16_disp32[] =
+  {0x66,0xe9};				/* jmp disp32	       */
 /* 32-bit NOPs patterns.  */
 static const unsigned char *const f32_patt[] = {
-  f32_1, f32_2, f32_3, f32_4, f32_5, f32_6, f32_7, f32_8,
-  f32_9, f32_10, f32_11, f32_12, f32_13, f32_14
+  f32_1, f32_2, f32_3, f32_4, NULL, f32_6, f32_7
 };
 /* 16-bit NOPs patterns.  */
 static const unsigned char *const f16_patt[] = {
-  f32_1, f32_2, f16_3, f16_4, f16_5, f16_6, f16_7, f16_8
+  f32_1, f32_2, f16_3, f16_4
 };
 /* nopl (%[re]ax) */
 static const unsigned char alt_3[] =
@@ -1261,18 +1225,13 @@ static const unsigned char alt_9[] =
 /* nopw %cs:0L(%[re]ax,%[re]ax,1) */
 static const unsigned char alt_10[] =
   {0x66,0x2e,0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00};
+/* data16 nopw %cs:0L(%eax,%eax,1) */
+static const unsigned char alt_11[] =
+  {0x66,0x66,0x2e,0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00};
 /* 32-bit and 64-bit NOPs patterns.  */
 static const unsigned char *const alt_patt[] = {
   f32_1, f32_2, alt_3, alt_4, alt_5, alt_6, alt_7, alt_8,
-  alt_9, alt_10
-};
-/* 64-bit only: nopw %cs:0L(%eax,%eax,1) */
-static const unsigned char alt64_11[] =
-  {0x67,0x66,0x2e,0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00};
-/* 64-bit NOPs patterns.  */
-static const unsigned char *const alt64_patt[] = {
-  f32_1, f32_2, alt_3, alt_4, alt_5, alt_6, alt_7, alt_8,
-  alt_9, alt_10, alt64_11
+  alt_9, alt_10, alt_11
 };
 
 /* Genenerate COUNT bytes of NOPs to WHERE from PATT with the maximum
@@ -1283,63 +1242,73 @@ i386_output_nops (char *where, const unsigned char *const *patt,
 		  int count, int max_single_nop_size)
 
 {
-  while (count > max_single_nop_size)
+  /* Place the longer NOP first.  */
+  int last;
+  int offset;
+  const unsigned char *nops =  patt[max_single_nop_size - 1];
+
+  /* Use the smaller one if the requsted one isn't available.  */
+  if (nops == NULL)
+    {
+      max_single_nop_size--;
+      nops = patt[max_single_nop_size - 1];
+    }
+
+  last = count % max_single_nop_size;
+
+  count -= last;
+  for (offset = 0; offset < count; offset += max_single_nop_size)
+    memcpy (where + offset, nops, max_single_nop_size);
+
+  if (last)
     {
-      count -= max_single_nop_size;
-      memcpy (where + count, patt[max_single_nop_size - 1],
-	      max_single_nop_size);
+      nops = patt[last - 1];
+      if (nops == NULL)
+	{
+	  /* Use the smaller one plus one-byte NOP if the needed one
+	     isn't available.  */
+	  last--;
+	  nops = patt[last - 1];
+	  memcpy (where + offset, nops, last);
+	  where[offset + last] = *patt[0];
+	}
+      else
+	memcpy (where + offset, nops, last);
     }
+}
 
-  if (count)
-    memcpy (where, patt[count - 1], count);
+static INLINE int
+fits_in_imm7 (offsetT num)
+{
+  return (num & 0x7f) == num;
 }
 
+static INLINE int
+fits_in_imm31 (offsetT num)
+{
+  return (num & 0x7fffffff) == num;
+}
 
 /* Genenerate COUNT bytes of NOPs to WHERE with the maximum size of a
    single NOP instruction LIMIT.  */
 
 void
-i386_generate_nops (fragS *f, char *where, offsetT count, int limit)
+i386_generate_nops (fragS *fragP, char *where, offsetT count, int limit)
 {
-  /* Output NOPs for .nop directive.  */
+  const unsigned char *const *patt = NULL;
   int max_single_nop_size;
-  const unsigned char *const *patt;
+  /* Maximum number of NOPs before switching to jump over NOPs.  */
+  int max_number_of_nops;
 
-  if (flag_code == CODE_16BIT)
-    {
-      patt = f16_patt;
-      max_single_nop_size = sizeof (f16_patt) / sizeof (f16_patt[0]);
-    }
-  else if (flag_code == CODE_64BIT)
-    {
-      patt = alt64_patt;
-      max_single_nop_size = sizeof (alt64_patt) / sizeof (alt64_patt[0]);
-    }
-  else
-    {
-      patt = alt_patt;
-      max_single_nop_size = sizeof (alt_patt) / sizeof (alt_patt[0]);
-    }
-  if (limit == 0)
-    limit = max_single_nop_size;
-  else if (limit > max_single_nop_size)
+  switch (fragP->fr_type)
     {
-      as_bad_where (f->fr_file, f->fr_line,
-		    _("invalide single nop size: %d (expect within [0, %d])"),
-		    limit, max_single_nop_size);
+    case rs_fill_nop:
+    case rs_align_code:
+      break;
+    default:
       return;
     }
 
-  i386_output_nops (where, patt, count, limit);
-}
-
-void
-i386_align_code (fragS *fragP, int count)
-{
-  /* Only align for at least a positive non-zero boundary. */
-  if (count <= 0 || count > MAX_MEM_FOR_RS_ALIGN_CODE)
-    return;
-
   /* We need to decide which NOP sequence to use for 32bit and
      64bit. When -mtune= is used:
 
@@ -1356,21 +1325,13 @@ i386_align_code (fragS *fragP, int count)
 
   if (flag_code == CODE_16BIT)
     {
-      if (count > 8)
-	{
-	  memcpy (fragP->fr_literal + fragP->fr_fix,
-		  jump_31, count);
-	  /* Adjust jump offset.  */
-	  fragP->fr_literal[fragP->fr_fix + 1] = count - 2;
-	}
-      else
-	memcpy (fragP->fr_literal + fragP->fr_fix,
-		f16_patt[count - 1], count);
+      patt = f16_patt;
+      max_single_nop_size = sizeof (f16_patt) / sizeof (f16_patt[0]);
+      /* Limit number of NOPs to 2 in 16-bit mode.  */
+      max_number_of_nops = 2;
     }
   else
     {
-      const unsigned char *const *patt = NULL;
-
       if (fragP->tc_frag_data.isa == PROCESSOR_UNKNOWN)
 	{
 	  /* PROCESSOR_UNKNOWN means that all ISAs may be used.  */
@@ -1461,38 +1422,79 @@ i386_align_code (fragS *fragP, int count)
 
       if (patt == f32_patt)
 	{
-	  /* If the padding is less than 15 bytes, we use the normal
-	     ones.  Otherwise, we use a jump instruction and adjust
-	     its offset.   */
-	  int limit;
+	  max_single_nop_size = sizeof (f32_patt) / sizeof (f32_patt[0]);
+	  /* Limit number of NOPs to 2 for older processors.  */
+	  max_number_of_nops = 2;
+	}
+      else
+	{
+	  max_single_nop_size = sizeof (alt_patt) / sizeof (alt_patt[0]);
+	  /* Limit number of NOPs to 7 for newer processors.  */
+	  max_number_of_nops = 7;
+	}
+    }
 
-	  /* For 64bit, the limit is 3 bytes.  */
-	  if (flag_code == CODE_64BIT
-	      && fragP->tc_frag_data.isa_flags.bitfield.cpulm)
-	    limit = 3;
-	  else
-	    limit = 15;
-	  if (count < limit)
-	    memcpy (fragP->fr_literal + fragP->fr_fix,
-		    patt[count - 1], count);
-	  else
-	    {
-	      memcpy (fragP->fr_literal + fragP->fr_fix,
-		      jump_31, count);
-	      /* Adjust jump offset.  */
-	      fragP->fr_literal[fragP->fr_fix + 1] = count - 2;
-	    }
+  if (limit == 0)
+    limit = max_single_nop_size;
+
+  if (fragP->fr_type == rs_fill_nop)
+    {
+      /* Output NOPs for .nop directive.  */
+      if (limit > max_single_nop_size)
+	{
+	  as_bad_where (fragP->fr_file, fragP->fr_line,
+			_("invalid single nop size: %d "
+			  "(expect within [0, %d])"),
+			limit, max_single_nop_size);
+	  return;
+	}
+    }
+  else
+    fragP->fr_var = count;
+
+  if ((count / max_single_nop_size) > max_number_of_nops)
+    {
+      /* Generate jump over NOPs.  */
+      offsetT disp = count - 2;
+      if (fits_in_imm7 (disp))
+	{
+	  /* Use "jmp disp8" if possible.  */
+	  count = disp;
+	  where[0] = jump_disp8[0];
+	  where[1] = count;
+	  where += 2;
 	}
       else
 	{
-	  /* Maximum length of an instruction is 10 byte.  If the
-	     padding is greater than 10 bytes and we don't use jump,
-	     we have to break it into smaller pieces.  */
-	  i386_output_nops (fragP->fr_literal + fragP->fr_fix,
-			    patt, count, 10);
+	  unsigned int size_of_jump;
+
+	  if (flag_code == CODE_16BIT)
+	    {
+	      where[0] = jump16_disp32[0];
+	      where[1] = jump16_disp32[1];
+	      size_of_jump = 2;
+	    }
+	  else
+	    {
+	      where[0] = jump32_disp32[0];
+	      size_of_jump = 1;
+	    }
+
+	  count -= size_of_jump + 4;
+	  if (!fits_in_imm31 (count))
+	    {
+	      as_bad_where (fragP->fr_file, fragP->fr_line,
+			    _("jump over nop padding out of range"));
+	      return;
+	    }
+
+	  md_number_to_chars (where + size_of_jump, count, 4);
+	  where += size_of_jump + 4;
 	}
     }
-  fragP->fr_var = count;
+
+  /* Generate multiple NOPs.  */
+  i386_output_nops (where, patt, count, limit);
 }
 
 static INLINE int
@@ -2204,18 +2206,6 @@ fits_in_imm4 (offsetT num)
   return (num & 0xf) == num;
 }
 
-static INLINE int
-fits_in_imm7 (offsetT num)
-{
-  return (num & 0x7f) == num;
-}
-
-static INLINE int
-fits_in_imm31 (offsetT num)
-{
-  return (num & 0x7fffffff) == num;
-}
-
 static i386_operand_type
 smallest_imm_type (offsetT num)
 {
diff --git a/gas/config/tc-i386.h b/gas/config/tc-i386.h
index 1250bc2..c7c53be 100644
--- a/gas/config/tc-i386.h
+++ b/gas/config/tc-i386.h
@@ -205,15 +205,7 @@ if ((n)									\
     goto around;							\
   }
 
-#define MAX_MEM_FOR_RS_ALIGN_CODE  31
-
-extern void i386_align_code (fragS *, int);
-
-#define HANDLE_ALIGN(fragP)						\
-if (fragP->fr_type == rs_align_code) 					\
-  i386_align_code (fragP, (fragP->fr_next->fr_address			\
-			   - fragP->fr_address				\
-			   - fragP->fr_fix));
+#define MAX_MEM_FOR_RS_ALIGN_CODE 4095
 
 void i386_print_statistics (FILE *);
 #define tc_print_statistics i386_print_statistics
@@ -286,6 +278,17 @@ extern void i386_generate_nops (fragS *, char *, offsetT, int);
 #define md_generate_nops(frag, where, amount, control) \
   i386_generate_nops ((frag), (where), (amount), (control))
 
+#define HANDLE_ALIGN(fragP)						\
+if (fragP->fr_type == rs_align_code) 					\
+  {									\
+    offsetT __count = (fragP->fr_next->fr_address			\
+		       - fragP->fr_address				\
+		       - fragP->fr_fix);				\
+    if (__count > 0 && __count <= MAX_MEM_FOR_RS_ALIGN_CODE)		\
+      md_generate_nops (fragP, fragP->fr_literal + fragP->fr_fix,	\
+			__count, 0);					\
+  }
+
 /* We want .cfi_* pseudo-ops for generating unwind info.  */
 #define TARGET_USE_CFIPOP 1