x86: shrink Masking insn attribute to a single bit (boolean)

The logic can actually be expressed with less code that way, utilizing
that there are common patterns of when which form of masking is
permitted. This then also eliminates the large set of open-codings of
BOTH_MASKING in the opcode table.

gas/config/tc-i386.c

@@ -6530,36 +6530,25 @@ check_VecOperands (const insn_template *t)
   /* Check if requested masking is supported.  */
   if (i.mask.reg)
     {
-      switch (t->opcode_modifier.masking)
+      if (!t->opcode_modifier.masking)
 	{
-	case BOTH_MASKING:
-	  break;
-	case MERGING_MASKING:
-	  if (i.mask.zeroing)
-	    {
-	case 0:
-	      i.error = unsupported_masking;
-	      return 1;
-	    }
-	  break;
-	case DYNAMIC_MASKING:
-	  /* Memory destinations allow only merging masking.  */
-	  if (i.mask.zeroing && i.mem_operands)
-	    {
-	      /* Find memory operand.  */
-	      for (op = 0; op < i.operands; op++)
-		if (i.flags[op] & Operand_Mem)
-		  break;
-	      gas_assert (op < i.operands);
-	      if (op == i.operands - 1)
-		{
-		  i.error = unsupported_masking;
-		  return 1;
-		}
-	    }
-	  break;
-	default:
-	  abort ();
+	  i.error = unsupported_masking;
+	  return 1;
+	}
+
+      /* Common rules for masking:
+	 - mask register destinations permit only zeroing-masking, without
+	   that actually being expressed by a {z} operand suffix or EVEX.z,
+	 - memory destinations allow only merging-masking,
+	 - scatter/gather insns (i.e. ones using vSIB) only allow merging-
+	   masking.  */
+      if (i.mask.zeroing
+	  && (t->operand_types[t->operands - 1].bitfield.class == RegMask
+	      || (i.flags[t->operands - 1] & Operand_Mem)
+	      || t->opcode_modifier.sib))
+	{
+	  i.error = unsupported_masking;
+	  return 1;
 	}
     }
 

opcodes/i386-opc.h

@@ -647,14 +647,7 @@ enum
 #define EVEX_L3                6
   EVex,
 
-  /* AVX512 masking support:
-	1: Zeroing or merging masking depending on operands.
-	2: Merging-masking.
-	3: Both zeroing and merging masking.
-   */
-#define DYNAMIC_MASKING 1
-#define MERGING_MASKING 2
-#define BOTH_MASKING    3
+  /* AVX512 masking support */
   Masking,
 
   /* AVX512 broadcast support.  The number of bytes to broadcast is
@@ -735,7 +728,7 @@ typedef struct i386_opcode_modifier
   unsigned int sib:3;
   unsigned int sse2avx:1;
   unsigned int evex:3;
-  unsigned int masking:2;
+  unsigned int masking:1;
   unsigned int broadcast:3;
   unsigned int staticrounding:1;
   unsigned int sae:1;