Check invalid mask registers

In 32-bit, the REX_B bit in the 3-byte VEX prefix is ignored and the
the highest bit in VEX.vvvv is either 1 or ignored.  In 64-bit, we
need to check invalid mask registers.

gas/

	PR binutis/20705
	* testsuite/gas/i386/i386.exp: Run x86-64-opcode-bad.
	* testsuite/gas/i386/x86-64-opcode-bad.d: New file.
	* testsuite/gas/i386/x86-64-opcode-bad.s: Likewise.

opcodes/

	PR binutis/20705
	* i386-dis.c (get_valid_dis386): Ignore the REX_B bit and
	the highest bit in VEX.vvvv for the 3-byte VEX prefix in
	32-bit mode.  Don't check vex.register_specifier in 32-bit
	mode.
	(OP_E_register): Check invalid mask registers.
	(OP_G): Likewise.
	(OP_VEX): Likewise.

6 files changed, 73 insertions, 17 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 2e3adbe..50282ab 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,10 @@
+2016-10-20  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR binutis/20705
+	* testsuite/gas/i386/i386.exp: Run x86-64-opcode-bad.
+	* testsuite/gas/i386/x86-64-opcode-bad.d: New file.
+	* testsuite/gas/i386/x86-64-opcode-bad.s: Likewise.
+
 2016-10-19  Renlin Li  <renlin.li@arm.com>
 
 	* config/tc-arm.c (encode_arm_shift): Generate unpredictable warning
diff --git a/gas/testsuite/gas/i386/i386.exp b/gas/testsuite/gas/i386/i386.exp
index c2cdf60..e2b874d 100644
--- a/gas/testsuite/gas/i386/i386.exp
+++ b/gas/testsuite/gas/i386/i386.exp
@@ -776,6 +776,7 @@ if [expr ([istarget "i*86-*-*"] || [istarget "x86_64-*-*"]) && [gas_64_check]] t
     run_dump_test "x86-64-pr20141"
     run_list_test "x86-64-avx512vl-1" "-al"
     run_list_test "x86-64-avx512vl-2" "-al"
+    run_dump_test "x86-64-opcode-bad"
 
     if { ![istarget "*-*-aix*"]
       && ![istarget "*-*-beos*"]
diff --git a/gas/testsuite/gas/i386/x86-64-opcode-bad.d b/gas/testsuite/gas/i386/x86-64-opcode-bad.d
new file mode 100644
index 0000000..24fb5eb
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-opcode-bad.d
@@ -0,0 +1,12 @@
+#as: --64
+#objdump: -dw
+#name: 64bit bad opcodes
+
+.*: +file format .*
+
+Disassembly of section .text:
+
+0+ <.text>:
+ +[a-f0-9]+:	c5 ac 46 f5          	kxnorw %k5,\(bad\),%k6
+ +[a-f0-9]+:	c5 2c 46 f5          	kxnorw %k5,\(bad\),\(bad\)
+#pass
diff --git a/gas/testsuite/gas/i386/x86-64-opcode-bad.s b/gas/testsuite/gas/i386/x86-64-opcode-bad.s
new file mode 100644
index 0000000..5acbcbb
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-opcode-bad.s
@@ -0,0 +1,10 @@
+	.text
+# All the followings are bad opcodes for x86-64.
+	.byte 0xc5
+	.byte 0xac
+	.byte 0x46
+	.byte 0xf5
+	.byte 0xc5
+	.byte 0x2c
+	.byte 0x46
+	.byte 0xf5
diff --git a/opcodes/ChangeLog b/opcodes/ChangeLog
index 36ed580..75c99ac 100644
--- a/opcodes/ChangeLog
+++ b/opcodes/ChangeLog
@@ -1,3 +1,12 @@
+2016-10-20  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR binutis/20705
+	* i386-dis.c (get_valid_dis386): Ignore the REX_B bit and
+	the highest bit in VEX.vvvv for the 3-byte VEX prefix in
+	32-bit mode.  Don't check vex.register_specifier in 32-bit
+	mode.
+	(OP_VEX): Check for invalid mask registers.
+
 2016-10-18  H.J. Lu  <hongjiu.lu@intel.com>
 
 	PR binutis/20699
diff --git a/opcodes/i386-dis.c b/opcodes/i386-dis.c
index 4624a6a..1518a1d 100644
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -13022,17 +13022,20 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
 	}
       codep++;
       vex.w = *codep & 0x80;
-      if (vex.w && address_mode == mode_64bit)
-	rex |= REX_W;
-
-      vex.register_specifier = (~(*codep >> 3)) & 0xf;
-      if (address_mode != mode_64bit
-	  && vex.register_specifier > 0x7)
+      if (address_mode == mode_64bit)
 	{
-	  dp = &bad_opcode;
-	  return dp;
+	  if (vex.w)
+	    rex |= REX_W;
+	  vex.register_specifier = (~(*codep >> 3)) & 0xf;
+	}
+      else
+	{
+	  /* For the 3-byte VEX prefix in 32-bit mode, the REX_B bit
+	     is ignored, other REX bits are 0 and the highest bit in
+	     VEX.vvvv is also ignored.  */
+	  rex = 0;
+	  vex.register_specifier = (~(*codep >> 3)) & 0x7;
 	}
-
       vex.length = (*codep & 0x4) ? 256 : 128;
       switch ((*codep & 0x3))
 	{
@@ -13072,16 +13075,10 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
       rex_ignored = rex;
       rex = (*codep & 0x80) ? 0 : REX_R;
 
+      /* For the 2-byte VEX prefix in 32-bit mode, the highest bit in
+	 VEX.vvvv is 1.  */
       vex.register_specifier = (~(*codep >> 3)) & 0xf;
-      if (address_mode != mode_64bit
-	  && vex.register_specifier > 0x7)
-	{
-	  dp = &bad_opcode;
-	  return dp;
-	}
-
       vex.w = 0;
-
       vex.length = (*codep & 0x4) ? 256 : 128;
       switch ((*codep & 0x3))
 	{
@@ -15266,6 +15263,11 @@ OP_E_register (int bytemode, int sizeflag)
       break;
     case mask_bd_mode:
     case mask_mode:
+      if (reg > 0x7)
+	{
+	  oappend ("(bad)");
+	  return;
+	}
       names = names_mask;
       break;
     case 0:
@@ -15795,6 +15797,11 @@ OP_G (int bytemode, int sizeflag)
       break;
     case mask_bd_mode:
     case mask_mode:
+      if ((modrm.reg + add) > 0x7)
+	{
+	  oappend ("(bad)");
+	  return;
+	}
       oappend (names_mask[modrm.reg + add]);
       break;
     default:
@@ -17225,6 +17232,11 @@ OP_VEX (int bytemode, int sizeflag ATTRIBUTE_UNUSED)
 	  break;
 	case mask_bd_mode:
 	case mask_mode:
+	  if (reg > 0x7)
+	    {
+	      oappend ("(bad)");
+	      return;
+	    }
 	  names = names_mask;
 	  break;
 	default:
@@ -17245,6 +17257,11 @@ OP_VEX (int bytemode, int sizeflag ATTRIBUTE_UNUSED)
 	  break;
 	case mask_bd_mode:
 	case mask_mode:
+	  if (reg > 0x7)
+	    {
+	      oappend ("(bad)");
+	      return;
+	    }
 	  names = names_mask;
 	  break;
 	default: