2 files changed, 114 insertions, 72 deletions

diff --git a/gcc/gimple.h b/gcc/gimple.h
index 3cde3cd..f7fdefc 100644
--- a/gcc/gimple.h
+++ b/gcc/gimple.h
@@ -4692,6 +4692,44 @@ gimple_phi_arg_has_location (const gphi *phi, size_t i)
   return gimple_phi_arg_location (phi, i) != UNKNOWN_LOCATION;
 }
 
+/* Return the number of arguments that can be accessed by gimple_arg.  */
+
+static inline unsigned
+gimple_num_args (const gimple *gs)
+{
+  if (auto phi = dyn_cast<const gphi *> (gs))
+    return gimple_phi_num_args (phi);
+  if (auto call = dyn_cast<const gcall *> (gs))
+    return gimple_call_num_args (call);
+  return gimple_num_ops (as_a <const gassign *> (gs)) - 1;
+}
+
+/* GS must be an assignment, a call, or a PHI.
+   If it's an assignment, return rhs operand I.
+   If it's a call, return function argument I.
+   If it's a PHI, return the value of PHI argument I.  */
+
+static inline tree
+gimple_arg (const gimple *gs, unsigned int i)
+{
+  if (auto phi = dyn_cast<const gphi *> (gs))
+    return gimple_phi_arg_def (phi, i);
+  if (auto call = dyn_cast<const gcall *> (gs))
+    return gimple_call_arg (call, i);
+  return gimple_op (as_a <const gassign *> (gs), i + 1);
+}
+
+/* Return a pointer to gimple_arg (GS, I).  */
+
+static inline tree *
+gimple_arg_ptr (gimple *gs, unsigned int i)
+{
+  if (auto phi = dyn_cast<gphi *> (gs))
+    return gimple_phi_arg_def_ptr (phi, i);
+  if (auto call = dyn_cast<gcall *> (gs))
+    return gimple_call_arg_ptr (call, i);
+  return gimple_op_ptr (as_a <gassign *> (gs), i + 1);
+}
 
 /* Return the region number for GIMPLE_RESX RESX_STMT.  */
 
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 410e586..2cd0b85 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -454,15 +454,57 @@ vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
 	      && (dtb == vect_external_def || dtb == vect_constant_def)));
 }
 
+static const int cond_expr_maps[3][5] = {
+  { 4, -1, -2, 1, 2 },
+  { 4, -2, -1, 1, 2 },
+  { 4, -1, -2, 2, 1 }
+};
+static const int arg2_map[] = { 1, 2 };
+
+/* For most SLP statements, there is a one-to-one mapping between
+   gimple arguments and child nodes.  If that is not true for STMT,
+   return an array that contains:
+
+   - the number of child nodes, followed by
+   - for each child node, the index of the argument associated with that node.
+     The special index -1 is the first operand of an embedded comparison and
+     the special index -2 is the second operand of an embedded comparison.
+
+   SWAP is as for vect_get_and_check_slp_defs.  */
+
+static const int *
+vect_get_operand_map (const gimple *stmt, unsigned char swap = 0)
+{
+  if (auto assign = dyn_cast<const gassign *> (stmt))
+    {
+      if (gimple_assign_rhs_code (assign) == COND_EXPR
+	  && COMPARISON_CLASS_P (gimple_assign_rhs1 (assign)))
+	return cond_expr_maps[swap];
+    }
+  gcc_assert (!swap);
+  if (auto call = dyn_cast<const gcall *> (stmt))
+    {
+      if (gimple_call_internal_p (call))
+	switch (gimple_call_internal_fn (call))
+	  {
+	  case IFN_MASK_LOAD:
+	    return arg2_map;
+
+	  default:
+	    break;
+	  }
+    }
+  return nullptr;
+}
+
 /* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
    they are of a valid type and that they match the defs of the first stmt of
    the SLP group (stored in OPRNDS_INFO).  This function tries to match stmts
-   by swapping operands of STMTS[STMT_NUM] when possible.  Non-zero *SWAP
-   indicates swap is required for cond_expr stmts.  Specifically, *SWAP
+   by swapping operands of STMTS[STMT_NUM] when possible.  Non-zero SWAP
+   indicates swap is required for cond_expr stmts.  Specifically, SWAP
    is 1 if STMT is cond and operands of comparison need to be swapped;
-   *SWAP is 2 if STMT is cond and code of comparison needs to be inverted.
-   If there is any operand swap in this function, *SWAP is set to non-zero
-   value.
+   SWAP is 2 if STMT is cond and code of comparison needs to be inverted.
+
    If there was a fatal error return -1; if the error could be corrected by
    swapping operands of father node of this one, return 1; if everything is
    ok return 0.  */
@@ -477,76 +519,48 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap,
   unsigned int i, number_of_oprnds;
   enum vect_def_type dt = vect_uninitialized_def;
   slp_oprnd_info oprnd_info;
-  int first_op_idx = 1;
   unsigned int commutative_op = -1U;
-  bool first_op_cond = false;
   bool first = stmt_num == 0;
 
+  if (!is_a<gcall *> (stmt_info->stmt)
+      && !is_a<gassign *> (stmt_info->stmt)
+      && !is_a<gphi *> (stmt_info->stmt))
+    return -1;
+
+  number_of_oprnds = gimple_num_args (stmt_info->stmt);
+  const int *map = vect_get_operand_map (stmt_info->stmt, swap);
+  if (map)
+    number_of_oprnds = *map++;
   if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
     {
-      number_of_oprnds = gimple_call_num_args (stmt);
-      first_op_idx = 3;
       if (gimple_call_internal_p (stmt))
 	{
 	  internal_fn ifn = gimple_call_internal_fn (stmt);
 	  commutative_op = first_commutative_argument (ifn);
-
-	  /* Masked load, only look at mask.  */
-	  if (ifn == IFN_MASK_LOAD)
-	    {
-	      number_of_oprnds = 1;
-	      /* Mask operand index.  */
-	      first_op_idx = 5;
-	    }
 	}
     }
   else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
     {
-      enum tree_code code = gimple_assign_rhs_code (stmt);
-      number_of_oprnds = gimple_num_ops (stmt) - 1;
-      /* Swap can only be done for cond_expr if asked to, otherwise we
-	 could result in different comparison code to the first stmt.  */
-      if (code == COND_EXPR
-	  && COMPARISON_CLASS_P (gimple_assign_rhs1 (stmt)))
-	{
-	  first_op_cond = true;
-	  number_of_oprnds++;
-	}
-      else
-	commutative_op = commutative_tree_code (code) ? 0U : -1U;
+      if (commutative_tree_code (gimple_assign_rhs_code (stmt)))
+	commutative_op = 0;
     }
-  else if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
-    number_of_oprnds = gimple_phi_num_args (stmt);
-  else
-    return -1;
 
   bool swapped = (swap != 0);
   bool backedge = false;
-  gcc_assert (!swapped || first_op_cond);
   enum vect_def_type *dts = XALLOCAVEC (enum vect_def_type, number_of_oprnds);
   for (i = 0; i < number_of_oprnds; i++)
     {
-      if (first_op_cond)
-	{
-	  /* Map indicating how operands of cond_expr should be swapped.  */
-	  int maps[3][4] = {{0, 1, 2, 3}, {1, 0, 2, 3}, {0, 1, 3, 2}};
-	  int *map = maps[swap];
-
-	  if (i < 2)
-	    oprnd = TREE_OPERAND (gimple_op (stmt_info->stmt,
-					     first_op_idx), map[i]);
-	  else
-	    oprnd = gimple_op (stmt_info->stmt, map[i]);
-	}
-      else if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
+      int opno = map ? map[i] : int (i);
+      if (opno < 0)
+	oprnd = TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
+      else
 	{
-	  oprnd = gimple_phi_arg_def (stmt, i);
-	  backedge = dominated_by_p (CDI_DOMINATORS,
-				     gimple_phi_arg_edge (stmt, i)->src,
-				     gimple_bb (stmt_info->stmt));
+	  oprnd = gimple_arg (stmt_info->stmt, opno);
+	  if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
+	    backedge = dominated_by_p (CDI_DOMINATORS,
+				       gimple_phi_arg_edge (stmt, opno)->src,
+				       gimple_bb (stmt_info->stmt));
 	}
-      else
-	oprnd = gimple_op (stmt_info->stmt, first_op_idx + (swapped ? !i : i));
       if (TREE_CODE (oprnd) == VIEW_CONVERT_EXPR)
 	oprnd = TREE_OPERAND (oprnd, 0);
 
@@ -1140,9 +1154,7 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
 
 	  if (need_same_oprnds)
 	    {
-	      tree other_op1 = (call_stmt
-				? gimple_call_arg (call_stmt, 1)
-				: gimple_assign_rhs2 (stmt));
+	      tree other_op1 = gimple_arg (stmt, 1);
 	      if (!operand_equal_p (first_op1, other_op1, 0))
 		{
 		  if (dump_enabled_p ())
@@ -1601,19 +1613,15 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
   matches[0] = false;
 
   stmt_vec_info stmt_info = stmts[0];
-  if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
-    nops = gimple_call_num_args (stmt);
-  else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
-    {
-      nops = gimple_num_ops (stmt) - 1;
-      if (gimple_assign_rhs_code (stmt) == COND_EXPR)
-	nops++;
-    }
-  else if (gphi *phi = dyn_cast <gphi *> (stmt_info->stmt))
-    nops = gimple_phi_num_args (phi);
-  else
+  if (!is_a<gcall *> (stmt_info->stmt)
+      && !is_a<gassign *> (stmt_info->stmt)
+      && !is_a<gphi *> (stmt_info->stmt))
     return NULL;
 
+  nops = gimple_num_args (stmt_info->stmt);
+  if (const int *map = vect_get_operand_map (stmt_info->stmt))
+    nops = map[0];
+
   /* If the SLP node is a PHI (induction or reduction), terminate
      the recursion.  */
   bool *skip_args = XALLOCAVEC (bool, nops);
@@ -1684,11 +1692,7 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
       && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
     {
       if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
-	{
-	  /* Masked load.  */
-	  gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
-	  nops = 1;
-	}
+	gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
       else
 	{
 	  *max_nunits = this_max_nunits;