x86/APX: correct .insn opcode space determination when REX2 is needed

In this case spaces 0f38 and 0f3a may not be put in place. To achieve
the intended effect, operand parsing (but not operand processing) needs
pulling ahead, so we know whether eGRP-s are in use.

4 files changed, 97 insertions, 28 deletions

diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 9da95c3..d169274 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -12922,13 +12922,43 @@ s_insn (int dummy ATTRIBUTE_UNUSED)
 	}
     }
 
+  /* Parse operands, if any, before evaluating encoding space.  */
+  if (*line == ',')
+    {
+      i.memshift = -1;
+
+      ptr = parse_operands (line + 1, &i386_mnemonics[MN__insn]);
+      this_operand = -1;
+      if (!ptr)
+	goto bad;
+      line = ptr;
+
+      if (!i.operands)
+	{
+	  as_bad (_("expecting operand after ','; got nothing"));
+	  goto done;
+	}
+
+      if (i.mem_operands > 1)
+	{
+	  as_bad (_("too many memory references for `%s'"),
+		  &i386_mnemonics[MN__insn]);
+	  goto done;
+	}
+
+      /* No need to distinguish encoding_evex and encoding_evex512.  */
+      if (i.encoding == encoding_evex512)
+	i.encoding = encoding_evex;
+    }
+
   /* Trim off encoding space.  */
   if (j > 1 && !i.insn_opcode_space && (val >> ((j - 1) * 8)) == 0x0f)
     {
       uint8_t byte = val >> ((--j - 1) * 8);
 
       i.insn_opcode_space = SPACE_0F;
-      switch (byte & -(j > 1))
+      switch (byte & -(j > 1 && !i.rex2_encoding
+		       && (i.encoding != encoding_egpr || evex)))
 	{
 	case 0x38:
 	  i.insn_opcode_space = SPACE_0F38;
@@ -12951,42 +12981,17 @@ s_insn (int dummy ATTRIBUTE_UNUSED)
   if (j > 2)
     {
       as_bad (_("opcode residual (%#"PRIx64") too wide"), (uint64_t) val);
-      goto bad;
+      goto done;
     }
   i.opcode_length = j;
 
   /* Handle operands, if any.  */
-  if (*line == ',')
+  if (i.operands)
     {
       i386_operand_type combined;
       expressionS *disp_exp = NULL;
       bool changed;
 
-      i.memshift = -1;
-
-      ptr = parse_operands (line + 1, &i386_mnemonics[MN__insn]);
-      this_operand = -1;
-      if (!ptr)
-	goto bad;
-      line = ptr;
-
-      if (!i.operands)
-	{
-	  as_bad (_("expecting operand after ','; got nothing"));
-	  goto done;
-	}
-
-      if (i.mem_operands > 1)
-	{
-	  as_bad (_("too many memory references for `%s'"),
-		  &i386_mnemonics[MN__insn]);
-	  goto done;
-	}
-
-      /* No need to distinguish encoding_evex and encoding_evex512.  */
-      if (i.encoding == encoding_evex512)
-	i.encoding = encoding_evex;
-
       if (i.encoding == encoding_egpr)
 	{
 	  if (vex || xop)
diff --git a/gas/testsuite/gas/i386/insn-rex2.l b/gas/testsuite/gas/i386/insn-rex2.l
new file mode 100644
index 0000000..d9e31bb
--- /dev/null
+++ b/gas/testsuite/gas/i386/insn-rex2.l
@@ -0,0 +1,38 @@
+[ 	]*[0-9]+[ 	]+\.text
+[ 	]*[0-9]+[ 	]+insn_rex2:
+[ 	]*[0-9]+ .... D58001C0[ 	]+\.insn \{rex2\} 0x0f01/0, %eax
+[ 	]*[0-9]+ .... D58038C0[ 	]+\.insn \{rex2\} 0x0f38/0, %eax
+[ 	]*[0-9]+ .... D5803801[ 	]+\.insn \{rex2\} 0x0f3801/0, %eax
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5803901[ 	]+\.insn \{rex2\} 0x0f3901/0, %eax
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5803A01[ 	]+\.insn \{rex2\} 0x0f3a01/0, \$0xCC, %eax
+[ 	]*[0-9]+ +C0CC
+[ 	]*[0-9]+[ 	]+
+[ 	]*[0-9]+ .... D58801C0[ 	]+\.insn \{rex2\} 0x0f01/0, %rax
+[ 	]*[0-9]+ .... D58838C0[ 	]+\.insn \{rex2\} 0x0f38/0, %rax
+[ 	]*[0-9]+ .... D5883801[ 	]+\.insn \{rex2\} 0x0f3801/0, %rax
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5883901[ 	]+\.insn \{rex2\} 0x0f3901/0, %rax
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5883A01[ 	]+\.insn \{rex2\} 0x0f3a01/0, \$0xCC, %rax
+[ 	]*[0-9]+ +C0CC
+[ 	]*[0-9]+[ 	]+
+[ 	]*[0-9]+ .... D58901C0[ 	]+\.insn \{rex2\} 0x0f01/0, %r8
+[ 	]*[0-9]+ .... D58938C0[ 	]+\.insn \{rex2\} 0x0f38/0, %r8
+[ 	]*[0-9]+ .... D5893801[ 	]+\.insn \{rex2\} 0x0f3801/0, %r8
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5893901[ 	]+\.insn \{rex2\} 0x0f3901/0, %r8
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5893A01[ 	]+\.insn \{rex2\} 0x0f3a01/0, \$0xCC, %r8
+[ 	]*[0-9]+ +C0CC
+[ 	]*[0-9]+[ 	]+
+[ 	]*[0-9]+ .... D59801C0[ 	]+\.insn 0x0f01/0, %r16
+[ 	]*[0-9]+ .... D59838C0[ 	]+\.insn 0x0f38/0, %r16
+[ 	]*[0-9]+ .... D5983801[ 	]+\.insn 0x0f3801/0, %r16
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5983901[ 	]+\.insn 0x0f3901/0, %r16
+[ 	]*[0-9]+ +C0
+[ 	]*[0-9]+ .... D5983A01[ 	]+\.insn 0x0f3a01/0, \$0xCC, %r16
+[ 	]*[0-9]+[ 	]+C0CC
+#pass
diff --git a/gas/testsuite/gas/i386/insn-rex2.s b/gas/testsuite/gas/i386/insn-rex2.s
new file mode 100644
index 0000000..b93159a
--- /dev/null
+++ b/gas/testsuite/gas/i386/insn-rex2.s
@@ -0,0 +1,25 @@
+	.text
+insn_rex2:
+	.insn {rex2} 0x0f01/0, %eax
+	.insn {rex2} 0x0f38/0, %eax
+	.insn {rex2} 0x0f3801/0, %eax
+	.insn {rex2} 0x0f3901/0, %eax
+	.insn {rex2} 0x0f3a01/0, $0xCC, %eax
+
+	.insn {rex2} 0x0f01/0, %rax
+	.insn {rex2} 0x0f38/0, %rax
+	.insn {rex2} 0x0f3801/0, %rax
+	.insn {rex2} 0x0f3901/0, %rax
+	.insn {rex2} 0x0f3a01/0, $0xCC, %rax
+
+	.insn {rex2} 0x0f01/0, %r8
+	.insn {rex2} 0x0f38/0, %r8
+	.insn {rex2} 0x0f3801/0, %r8
+	.insn {rex2} 0x0f3901/0, %r8
+	.insn {rex2} 0x0f3a01/0, $0xCC, %r8
+
+	.insn 0x0f01/0, %r16
+	.insn 0x0f38/0, %r16
+	.insn 0x0f3801/0, %r16
+	.insn 0x0f3901/0, %r16
+	.insn 0x0f3a01/0, $0xCC, %r16
diff --git a/gas/testsuite/gas/i386/x86-64.exp b/gas/testsuite/gas/i386/x86-64.exp
index f0ca846..e53c135 100644
--- a/gas/testsuite/gas/i386/x86-64.exp
+++ b/gas/testsuite/gas/i386/x86-64.exp
@@ -126,6 +126,7 @@ run_dump_test "x86-64-sysenter-mixed"
 run_dump_test "x86-64-sysenter-amd"
 run_list_test "x86-64-sysenter-amd" "-mamd64"
 run_dump_test "insn-64"
+run_list_test "insn-rex2" "-aln --divide"
 run_dump_test "noreg64"
 run_list_test "noreg64"
 run_dump_test "noreg64-data16"