x86: improve a few diagnostics

PR gas/31043
"unsupported instruction ..." can mean about anything, and can also be
mistaken to mean something that isn't meant. Replace most of its uses by
more specific diagnostics,

While there also take the opportunity and purge the no longer used
invalid_register_operand enumerator.

9 files changed, 93 insertions, 84 deletions

diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 1692dfc..2651cd2 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -242,6 +242,8 @@ enum i386_error
     unsupported,
     unsupported_on_arch,
     unsupported_64bit,
+    no_vex_encoding,
+    no_evex_encoding,
     invalid_sib_address,
     invalid_vsib_address,
     invalid_vector_register_set,
@@ -254,7 +256,7 @@ enum i386_error
     mask_not_on_destination,
     no_default_mask,
     unsupported_rc_sae,
-    invalid_register_operand,
+    unsupported_vector_size,
     internal_error,
   };
 
@@ -5304,6 +5306,12 @@ md_assemble (char *line)
 			pass1_mnem ? pass1_mnem : insn_name (current_templates->start));
 	    }
 	  return;
+	case no_vex_encoding:
+	  err_msg = _("no VEX/XOP encoding");
+	  break;
+	case no_evex_encoding:
+	  err_msg = _("no EVEX encoding");
+	  break;
 	case invalid_sib_address:
 	  err_msg = _("invalid SIB address");
 	  break;
@@ -5340,9 +5348,10 @@ md_assemble (char *line)
 	case unsupported_rc_sae:
 	  err_msg = _("unsupported static rounding/sae");
 	  break;
-	case invalid_register_operand:
-	  err_msg = _("invalid register operand");
-	  break;
+	case unsupported_vector_size:
+	  as_bad (_("vector size above %u required for `%s'"), 128u << vector_size,
+		  pass1_mnem ? pass1_mnem : insn_name (current_templates->start));
+	  return;
 	case internal_error:
 	  err_msg = _("internal error");
 	  break;
@@ -6567,7 +6576,7 @@ check_VecOperands (const insn_template *t)
 	      && (i.types[op].bitfield.ymmword
 		  || i.types[op].bitfield.xmmword))
 	    {
-	      i.error = unsupported;
+	      i.error = operand_size_mismatch;
 	      return 1;
 	    }
 	}
@@ -6583,7 +6592,7 @@ check_VecOperands (const insn_template *t)
 	  if (t->operand_types[op].bitfield.xmmword
 	      && i.types[op].bitfield.ymmword)
 	    {
-	      i.error = unsupported;
+	      i.error = operand_size_mismatch;
 	      return 1;
 	    }
 	}
@@ -6963,7 +6972,7 @@ VEX_check_encoding (const insn_template *t)
 	      || t->opcode_modifier.vex == VEX256
 	      || t->opcode_modifier.vsz >= VSZ256)))
     {
-      i.error = unsupported;
+      i.error = unsupported_vector_size;
       return 1;
     }
 
@@ -6973,7 +6982,7 @@ VEX_check_encoding (const insn_template *t)
       /* This instruction must be encoded with EVEX prefix.  */
       if (!t->opcode_modifier.evex)
 	{
-	  i.error = unsupported;
+	  i.error = no_evex_encoding;
 	  return 1;
 	}
       return 0;
@@ -6984,7 +6993,7 @@ VEX_check_encoding (const insn_template *t)
       /* This instruction template doesn't have VEX prefix.  */
       if (i.vec_encoding != vex_encoding_default)
 	{
-	  i.error = unsupported;
+	  i.error = no_vex_encoding;
 	  return 1;
 	}
       return 0;
diff --git a/gas/testsuite/gas/i386/avx10-vsz.l b/gas/testsuite/gas/i386/avx10-vsz.l
index 8cbc2b2..d10ff55 100644
--- a/gas/testsuite/gas/i386/avx10-vsz.l
+++ b/gas/testsuite/gas/i386/avx10-vsz.l
@@ -4,71 +4,71 @@
 .*:47: Error: ambiguous operand size for `vcvtpd2ps'
 .*:48: Error: ambiguous operand size for `vfpclassps'
 .*:7: Warning: \.avx10\.1/256
-.*:12: Error: unsupported instruction `kunpckdq'
-.*:16: Error: unsupported instruction `kaddq'
-.*:17: Error: unsupported instruction `kandq'
-.*:18: Error: unsupported instruction `kandnq'
-.*:19: Error: unsupported instruction `kmovq'
-.*:20: Error: unsupported instruction `kmovq'
-.*:21: Error: unsupported instruction `kmovq'
-.*:22: Error: unsupported instruction `knotq'
-.*:23: Error: unsupported instruction `korq'
-.*:24: Error: unsupported instruction `kortestq'
-.*:25: Error: unsupported instruction `kshiftlq'
-.*:26: Error: unsupported instruction `kshiftrq'
-.*:27: Error: unsupported instruction `ktestq'
-.*:28: Error: unsupported instruction `kxorq'
-.*:29: Error: unsupported instruction `kxnorq'
-.*:32: Error: unsupported instruction `vcvtpd2phz'
+.*:12: Error: vector size .* `kunpckdq'
+.*:16: Error: vector size .* `kaddq'
+.*:17: Error: vector size .* `kandq'
+.*:18: Error: vector size .* `kandnq'
+.*:19: Error: vector size .* `kmovq'
+.*:20: Error: vector size .* `kmovq'
+.*:21: Error: vector size .* `kmovq'
+.*:22: Error: vector size .* `knotq'
+.*:23: Error: vector size .* `korq'
+.*:24: Error: vector size .* `kortestq'
+.*:25: Error: vector size .* `kshiftlq'
+.*:26: Error: vector size .* `kshiftrq'
+.*:27: Error: vector size .* `ktestq'
+.*:28: Error: vector size .* `kxorq'
+.*:29: Error: vector size .* `kxnorq'
+.*:32: Error: vector size .* `vcvtpd2phz'
 .*:34: Error: unsupported broadcast for `vcvtpd2ph'
-.*:37: Error: unsupported instruction `vcvtpd2ps'
-.*:43: Error: unsupported instruction `vfpclasspsz'
+.*:37: Error: vector size .* for `vcvtpd2ps'
+.*:43: Error: vector size .* `vfpclasspsz'
 .*:46: Error: ambiguous operand size for `vcvtpd2ph'
 .*:47: Error: ambiguous operand size for `vcvtpd2ps'
 .*:48: Error: ambiguous operand size for `vfpclassps'
 .*:7: Warning: \.avx10\.1/128
-.*:10: Error: unsupported instruction `kmovd'
-.*:11: Error: unsupported instruction `kmovd'
-.*:12: Error: unsupported instruction `kunpckdq'
-.*:13: Error: unsupported instruction `kunpckwd'
-.*:16: Error: unsupported instruction `kaddd'
-.*:17: Error: unsupported instruction `kandd'
-.*:18: Error: unsupported instruction `kandnd'
-.*:19: Error: unsupported instruction `kmovd'
-.*:20: Error: unsupported instruction `kmovd'
-.*:21: Error: unsupported instruction `kmovd'
-.*:22: Error: unsupported instruction `knotd'
-.*:23: Error: unsupported instruction `kord'
-.*:24: Error: unsupported instruction `kortestd'
-.*:25: Error: unsupported instruction `kshiftld'
-.*:26: Error: unsupported instruction `kshiftrd'
-.*:27: Error: unsupported instruction `ktestd'
-.*:28: Error: unsupported instruction `kxord'
-.*:29: Error: unsupported instruction `kxnord'
-.*:16: Error: unsupported instruction `kaddq'
-.*:17: Error: unsupported instruction `kandq'
-.*:18: Error: unsupported instruction `kandnq'
-.*:19: Error: unsupported instruction `kmovq'
-.*:20: Error: unsupported instruction `kmovq'
-.*:21: Error: unsupported instruction `kmovq'
-.*:22: Error: unsupported instruction `knotq'
-.*:23: Error: unsupported instruction `korq'
-.*:24: Error: unsupported instruction `kortestq'
-.*:25: Error: unsupported instruction `kshiftlq'
-.*:26: Error: unsupported instruction `kshiftrq'
-.*:27: Error: unsupported instruction `ktestq'
-.*:28: Error: unsupported instruction `kxorq'
-.*:29: Error: unsupported instruction `kxnorq'
-.*:32: Error: unsupported instruction `vcvtpd2phz'
-.*:33: Error: unsupported instruction `vcvtpd2phy'
+.*:10: Error: vector size .* `kmovd'
+.*:11: Error: vector size .* `kmovd'
+.*:12: Error: vector size .* `kunpckdq'
+.*:13: Error: vector size .* `kunpckwd'
+.*:16: Error: vector size .* `kaddd'
+.*:17: Error: vector size .* `kandd'
+.*:18: Error: vector size .* `kandnd'
+.*:19: Error: vector size .* `kmovd'
+.*:20: Error: vector size .* `kmovd'
+.*:21: Error: vector size .* `kmovd'
+.*:22: Error: vector size .* `knotd'
+.*:23: Error: vector size .* `kord'
+.*:24: Error: vector size .* `kortestd'
+.*:25: Error: vector size .* `kshiftld'
+.*:26: Error: vector size .* `kshiftrd'
+.*:27: Error: vector size .* `ktestd'
+.*:28: Error: vector size .* `kxord'
+.*:29: Error: vector size .* `kxnord'
+.*:16: Error: vector size .* `kaddq'
+.*:17: Error: vector size .* `kandq'
+.*:18: Error: vector size .* `kandnq'
+.*:19: Error: vector size .* `kmovq'
+.*:20: Error: vector size .* `kmovq'
+.*:21: Error: vector size .* `kmovq'
+.*:22: Error: vector size .* `knotq'
+.*:23: Error: vector size .* `korq'
+.*:24: Error: vector size .* `kortestq'
+.*:25: Error: vector size .* `kshiftlq'
+.*:26: Error: vector size .* `kshiftrq'
+.*:27: Error: vector size .* `ktestq'
+.*:28: Error: vector size .* `kxorq'
+.*:29: Error: vector size .* `kxnorq'
+.*:32: Error: vector size .* `vcvtpd2phz'
+.*:33: Error: vector size .* `vcvtpd2phy'
 .*:34: Error: unsupported broadcast for `vcvtpd2ph'
 .*:35: Error: unsupported broadcast for `vcvtpd2ph'
 .*:37: Error: .*
-.*:38: Error: unsupported instruction `vcvtpd2psy'
-.*:39: Error: unsupported instruction `vcvtpd2psy'
+.*:38: Error: vector size .* `vcvtpd2psy'
+.*:39: Error: vector size .* `vcvtpd2psy'
 .*:40: Error: unsupported broadcast for `vcvtpd2ps'
-.*:42: Error: unsupported instruction `vfpclasspsy'
-.*:43: Error: unsupported instruction `vfpclasspsz'
+.*:42: Error: vector size .* `vfpclasspsy'
+.*:43: Error: vector size .* `vfpclasspsz'
 .*:7: Warning: \.avx10\.1
 .*:46: Error: ambiguous operand size for `vcvtpd2ph'
 .*:47: Error: ambiguous operand size for `vcvtpd2ps'
diff --git a/gas/testsuite/gas/i386/avx512vl-1.l b/gas/testsuite/gas/i386/avx512vl-1.l
index 183c21d..e247a10 100644
--- a/gas/testsuite/gas/i386/avx512vl-1.l
+++ b/gas/testsuite/gas/i386/avx512vl-1.l
@@ -3,8 +3,8 @@
 .*:8: Error: .*corei7\.avx.*
 .*:9: Error: .*corei7\.avx.*
 .*:10: Error: .*corei7\.avx.*
-.*:15: Error: .*unsupported.*
-.*:16: Error: .*unsupported.*
+.*:15: Error: .*operand size mismatch.*
+.*:16: Error: .*operand size mismatch.*
 GAS LISTING .*
 #...
 [ 	]*1[ 	]+\.text
diff --git a/gas/testsuite/gas/i386/inval-avx512vl.l b/gas/testsuite/gas/i386/inval-avx512vl.l
index b0c0399..cdd1f19 100644
--- a/gas/testsuite/gas/i386/inval-avx512vl.l
+++ b/gas/testsuite/gas/i386/inval-avx512vl.l
@@ -1,6 +1,6 @@
 .*: Assembler messages:
-.*:5: Error: .*unsupported instruction.*
-.*:6: Error: .*unsupported instruction.*
+.*:5: Error: .*operand size mismatch.*
+.*:6: Error: .*operand size mismatch.*
 GAS LISTING .*
 
 
diff --git a/gas/testsuite/gas/i386/noavx512-2.l b/gas/testsuite/gas/i386/noavx512-2.l
index 3f9b215..25d18d0 100644
--- a/gas/testsuite/gas/i386/noavx512-2.l
+++ b/gas/testsuite/gas/i386/noavx512-2.l
@@ -5,7 +5,7 @@
 .*:17: Error: .*bad register name `%zmm.*
 .*:18: Error: .*bad register name `%zmm.*
 .*:21: Error: .*bad register name `%zmm.*
-.*:24: Error: .*unsupported instruction.*
+.*:24: Error: .*vector size.*
 .*:25: Error: .*bad register name `%zmm.*
 .*:8: Error: .*bad register name `%zmm.*
 .*:10: Error: .*bad register name `%ymm.*
@@ -23,16 +23,16 @@
 .*:27: Error: .*bad register name `%ymm.*
 .*:9: Error: .*unsupported masking.*
 .*:10: Error: .*unsupported masking.*
-.*:12: Error: .*unsupported instruction.*
-.*:13: Error: .*unsupported instruction.*
-.*:15: Error: .*unsupported instruction.*
-.*:16: Error: .*unsupported instruction.*
+.*:12: Error: .*operand size mismatch.*
+.*:13: Error: .*operand size mismatch.*
+.*:15: Error: .*operand size mismatch.*
+.*:16: Error: .*operand size mismatch.*
 .*:19: Error: .*unsupported masking.*
 .*:20: Error: .*unsupported masking.*
-.*:22: Error: .*unsupported instruction.*
-.*:23: Error: .*unsupported instruction.*
-.*:26: Error: .*unsupported instruction.*
-.*:27: Error: .*unsupported instruction.*
+.*:22: Error: .*operand size mismatch.*
+.*:23: Error: .*operand size mismatch.*
+.*:26: Error: .*operand size mismatch.*
+.*:27: Error: .*operand size mismatch.*
 #...
 [ 	]*[0-9]+[ 	]+\# Test \.arch \.noavx512vl
 [ 	]*[0-9]+[ 	]+\.text
diff --git a/gas/testsuite/gas/i386/x86-64-avx-ifma-inval.l b/gas/testsuite/gas/i386/x86-64-avx-ifma-inval.l
index fad43f6..3300695 100644
--- a/gas/testsuite/gas/i386/x86-64-avx-ifma-inval.l
+++ b/gas/testsuite/gas/i386/x86-64-avx-ifma-inval.l
@@ -1,4 +1,4 @@
 .* Assembler messages:
 .*:6: Error: unsupported .* `vpmadd52huq'
-.*:7: Error: unsupported .* `vpmadd52huq'
+.*:7: Error: no EVEX .* `vpmadd52huq'
 .*:8: Error: operand .* `vpmadd52huq'
diff --git a/gas/testsuite/gas/i386/x86-64-avx-vnni-inval.l b/gas/testsuite/gas/i386/x86-64-avx-vnni-inval.l
index 6180866..ca0f212 100644
--- a/gas/testsuite/gas/i386/x86-64-avx-vnni-inval.l
+++ b/gas/testsuite/gas/i386/x86-64-avx-vnni-inval.l
@@ -1,4 +1,4 @@
 .* Assembler messages:
 .*:6: Error: unsupported .* `vpdpbusds'
-.*:7: Error: unsupported .* `vpdpbusds'
+.*:7: Error: no EVEX .* `vpdpbusds'
 .*:8: Error: operand .* `vpdpbusds'
diff --git a/gas/testsuite/gas/i386/x86-64-avx512vl-1.l b/gas/testsuite/gas/i386/x86-64-avx512vl-1.l
index c5982e7..3dfab96 100644
--- a/gas/testsuite/gas/i386/x86-64-avx512vl-1.l
+++ b/gas/testsuite/gas/i386/x86-64-avx512vl-1.l
@@ -3,8 +3,8 @@
 .*:8: Error: .*corei7\.avx.*
 .*:9: Error: .*corei7\.avx.*
 .*:10: Error: .*corei7\.avx.*
-.*:15: Error: .*unsupported.*
-.*:16: Error: .*unsupported.*
+.*:15: Error: .*operand size mismatch.*
+.*:16: Error: .*operand size mismatch.*
 GAS LISTING .*
 #...
 [ 	]*1[ 	]+\.text
diff --git a/gas/testsuite/gas/i386/x86-64-inval-avx512vl.l b/gas/testsuite/gas/i386/x86-64-inval-avx512vl.l
index 7b661f3..a0db1cc 100644
--- a/gas/testsuite/gas/i386/x86-64-inval-avx512vl.l
+++ b/gas/testsuite/gas/i386/x86-64-inval-avx512vl.l
@@ -1,6 +1,6 @@
 .*: Assembler messages:
-.*:5: Error: .*unsupported instruction.*
-.*:6: Error: .*unsupported instruction.*
+.*:5: Error: .*operand size mismatch.*
+.*:6: Error: .*operand size mismatch.*
 GAS LISTING .*