openmp: Implement OpenMP 5.1 atomics, so far for C only

This patch implements OpenMP 5.1 atomics (with clarifications from upcoming 5.2).
The most important changes are that it is now possible to write (for C/C++,
for Fortran it was possible before already) min/max atomics and more importantly
compare and exchange in various forms.
Also, acq_rel is now allowed on read/write and acq_rel/acquire are allowed on
update, and there are new compare, weak and fail clauses.

2021-09-10  Jakub Jelinek  <jakub@redhat.com>

gcc/
	* tree-core.h (enum omp_memory_order): Add OMP_MEMORY_ORDER_MASK,
	OMP_FAIL_MEMORY_ORDER_UNSPECIFIED, OMP_FAIL_MEMORY_ORDER_RELAXED,
	OMP_FAIL_MEMORY_ORDER_ACQUIRE, OMP_FAIL_MEMORY_ORDER_RELEASE,
	OMP_FAIL_MEMORY_ORDER_ACQ_REL, OMP_FAIL_MEMORY_ORDER_SEQ_CST and
	OMP_FAIL_MEMORY_ORDER_MASK enumerators.
	(OMP_FAIL_MEMORY_ORDER_SHIFT): Define.
	* gimple-pretty-print.c (dump_gimple_omp_atomic_load,
	dump_gimple_omp_atomic_store): Print [weak] for weak atomic
	load/store.
	* gimple.h (enum gf_mask): Change GF_OMP_ATOMIC_MEMORY_ORDER
	to 6-bit mask, adjust GF_OMP_ATOMIC_NEED_VALUE value and add
	GF_OMP_ATOMIC_WEAK.
	(gimple_omp_atomic_weak_p, gimple_omp_atomic_set_weak): New inline
	functions.
	* tree.h (OMP_ATOMIC_WEAK): Define.
	* tree-pretty-print.c (dump_omp_atomic_memory_order): Adjust for
	fail memory order being encoded in the same enum and also print
	fail clause if present.
	(dump_generic_node): Print weak clause if OMP_ATOMIC_WEAK.
	* gimplify.c (goa_stabilize_expr): Add target_expr and rhs arguments,
	handle pre_p == NULL case as a test mode that only returns value
	but doesn't change gimplify nor change anything otherwise, adjust
	recursive calls, add MODIFY_EXPR, ADDR_EXPR, COND_EXPR, TARGET_EXPR
	and CALL_EXPR handling, adjust COMPOUND_EXPR handling for
	__builtin_clear_padding calls, for !rhs gimplify as lvalue rather
	than rvalue.
	(gimplify_omp_atomic): Adjust goa_stabilize_expr caller.  Handle
	COND_EXPR rhs.  Set weak flag on gimple load/store for
	OMP_ATOMIC_WEAK.
	* omp-expand.c (omp_memory_order_to_fail_memmodel): New function.
	(omp_memory_order_to_memmodel): Adjust for fail clause encoded
	in the same enum.
	(expand_omp_atomic_cas): New function.
	(expand_omp_atomic_pipeline): Use omp_memory_order_to_fail_memmodel
	function.
	(expand_omp_atomic): Attempt to optimize atomic compare and exchange
	using expand_omp_atomic_cas.
gcc/c-family/
	* c-common.h (c_finish_omp_atomic): Add r and weak arguments.
	* c-omp.c: Include gimple-fold.h.
	(c_finish_omp_atomic): Add r and weak arguments.  Add support for
	OpenMP 5.1 atomics.
gcc/c/
	* c-parser.c (c_parser_conditional_expression): If omp_atomic_lhs and
	cond.value is >, < or == with omp_atomic_lhs as one of the operands,
	don't call build_conditional_expr, instead build a COND_EXPR directly.
	(c_parser_binary_expression): Avoid calling parser_build_binary_op
	if omp_atomic_lhs even in more cases for >, < or ==.
	(c_parser_omp_atomic): Update function comment for OpenMP 5.1 atomics,
	parse OpenMP 5.1 atomics and fail, compare and weak clauses, allow
	acq_rel on atomic read/write and acq_rel/acquire clauses on update.
	* c-typeck.c (build_binary_op): For flag_openmp only handle
	MIN_EXPR/MAX_EXPR.
gcc/cp/
	* parser.c (cp_parser_omp_atomic): Allow acq_rel on atomic read/write
	and acq_rel/acquire clauses on update.
	* semantics.c (finish_omp_atomic): Adjust c_finish_omp_atomic caller.
gcc/testsuite/
	* c-c++-common/gomp/atomic-17.c (foo): Add tests for atomic read,
	write or update with acq_rel clause and atomic update with acquire clause.
	* c-c++-common/gomp/atomic-18.c (foo): Adjust expected diagnostics
	wording, remove tests moved to atomic-17.c.
	* c-c++-common/gomp/atomic-21.c: Expect only 2 omp atomic release and
	2 omp atomic acq_rel directives instead of 4 omp atomic release.
	* c-c++-common/gomp/atomic-25.c: New test.
	* c-c++-common/gomp/atomic-26.c: New test.
	* c-c++-common/gomp/atomic-27.c: New test.
	* c-c++-common/gomp/atomic-28.c: New test.
	* c-c++-common/gomp/atomic-29.c: New test.
	* c-c++-common/gomp/atomic-30.c: New test.
	* c-c++-common/goacc-gomp/atomic.c: Expect 1 omp atomic release and
	1 omp atomic_acq_rel instead of 2 omp atomic release directives.
	* gcc.dg/gomp/atomic-5.c: Adjust expected error diagnostic wording.
	* g++.dg/gomp/atomic-18.C:Expect 4 omp atomic release and
	1 omp atomic_acq_rel instead of 5 omp atomic release directives.
libgomp/
	* testsuite/libgomp.c-c++-common/atomic-19.c: New test.
	* testsuite/libgomp.c-c++-common/atomic-20.c: New test.
	* testsuite/libgomp.c-c++-common/atomic-21.c: New test.

1 files changed, 308 insertions, 15 deletions

diff --git a/gcc/omp-expand.c b/gcc/omp-expand.c
index f2b6f6f..159ae0e 100644
--- a/gcc/omp-expand.c
+++ b/gcc/omp-expand.c
@@ -8487,22 +8487,58 @@ expand_omp_synch (struct omp_region *region)
     }
 }
 
+/* Translate enum omp_memory_order to enum memmodel for the embedded
+   fail clause in there.  */
+
+static enum memmodel
+omp_memory_order_to_fail_memmodel (enum omp_memory_order mo)
+{
+  switch (mo & OMP_FAIL_MEMORY_ORDER_MASK)
+    {
+    case OMP_FAIL_MEMORY_ORDER_UNSPECIFIED:
+      switch (mo & OMP_MEMORY_ORDER_MASK)
+	{
+	case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
+	case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
+	case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELAXED;
+	case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQUIRE;
+	case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
+	default: break;
+	}
+      gcc_unreachable ();
+    case OMP_FAIL_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
+    case OMP_FAIL_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
+    case OMP_FAIL_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
+    default: gcc_unreachable ();
+    }
+}
+
 /* Translate enum omp_memory_order to enum memmodel.  The two enums
    are using different numbers so that OMP_MEMORY_ORDER_UNSPECIFIED
-   is 0.  */
+   is 0 and omp_memory_order has the fail mode encoded in it too.  */
 
 static enum memmodel
 omp_memory_order_to_memmodel (enum omp_memory_order mo)
 {
-  switch (mo)
-    {
-    case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
-    case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
-    case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELEASE;
-    case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQ_REL;
-    case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
+  enum memmodel ret, fail_ret;
+  switch (mo & OMP_MEMORY_ORDER_MASK)
+    {
+    case OMP_MEMORY_ORDER_RELAXED: ret = MEMMODEL_RELAXED; break;
+    case OMP_MEMORY_ORDER_ACQUIRE: ret = MEMMODEL_ACQUIRE; break;
+    case OMP_MEMORY_ORDER_RELEASE: ret = MEMMODEL_RELEASE; break;
+    case OMP_MEMORY_ORDER_ACQ_REL: ret = MEMMODEL_ACQ_REL; break;
+    case OMP_MEMORY_ORDER_SEQ_CST: ret = MEMMODEL_SEQ_CST; break;
     default: gcc_unreachable ();
     }
+  /* If we drop the -Winvalid-memory-model warning for C++17 P0418R2,
+     we can just return ret here unconditionally.  Otherwise, work around
+     it here and make sure fail memmodel is not stronger.  */
+  if ((mo & OMP_FAIL_MEMORY_ORDER_MASK) == OMP_FAIL_MEMORY_ORDER_UNSPECIFIED)
+    return ret;
+  fail_ret = omp_memory_order_to_fail_memmodel (mo);
+  if (fail_ret > ret)
+    return fail_ret;
+  return ret;
 }
 
 /* A subroutine of expand_omp_atomic.  Attempt to implement the atomic
@@ -8782,6 +8818,261 @@ expand_omp_atomic_fetch_op (basic_block load_bb,
   return true;
 }
 
+/* A subroutine of expand_omp_atomic.  Attempt to implement the atomic
+   compare and exchange as an ATOMIC_COMPARE_EXCHANGE internal function.
+   Returns false if the expression is not of the proper form.  */
+
+static bool
+expand_omp_atomic_cas (basic_block load_bb, tree addr,
+		       tree loaded_val, tree stored_val, int index)
+{
+  /* We expect to find the following sequences:
+
+   load_bb:
+       GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
+
+   store_bb:
+       val = tmp == e ? d : tmp;
+       GIMPLE_OMP_ATOMIC_STORE (val)
+
+     or in store_bb instead:
+       tmp2 = tmp == e;
+       val = tmp2 ? d : tmp;
+       GIMPLE_OMP_ATOMIC_STORE (val)
+
+     or:
+       tmp3 = VIEW_CONVERT_EXPR<integral_type>(tmp);
+       val = e == tmp3 ? d : tmp;
+       GIMPLE_OMP_ATOMIC_STORE (val)
+
+     etc.  */
+
+
+  basic_block store_bb = single_succ (load_bb);
+  gimple_stmt_iterator gsi = gsi_last_nondebug_bb (store_bb);
+  gimple *store_stmt = gsi_stmt (gsi);
+  if (!store_stmt || gimple_code (store_stmt) != GIMPLE_OMP_ATOMIC_STORE)
+    return false;
+  gsi_prev_nondebug (&gsi);
+  if (gsi_end_p (gsi))
+    return false;
+  gimple *condexpr_stmt = gsi_stmt (gsi);
+  if (!is_gimple_assign (condexpr_stmt)
+      || gimple_assign_rhs_code (condexpr_stmt) != COND_EXPR)
+    return false;
+  if (!operand_equal_p (gimple_assign_lhs (condexpr_stmt), stored_val, 0))
+    return false;
+  gimple *cond_stmt = NULL;
+  gimple *vce_stmt = NULL;
+  gsi_prev_nondebug (&gsi);
+  if (!gsi_end_p (gsi))
+    {
+      cond_stmt = gsi_stmt (gsi);
+      if (!is_gimple_assign (cond_stmt))
+	return false;
+      if (gimple_assign_rhs_code (cond_stmt) == EQ_EXPR)
+	{
+	  gsi_prev_nondebug (&gsi);
+	  if (!gsi_end_p (gsi))
+	    {
+	      vce_stmt = gsi_stmt (gsi);
+	      if (!is_gimple_assign (vce_stmt)
+		  || gimple_assign_rhs_code (vce_stmt) != VIEW_CONVERT_EXPR)
+		return false;
+	    }
+	}
+      else if (gimple_assign_rhs_code (cond_stmt) == VIEW_CONVERT_EXPR)
+	std::swap (vce_stmt, cond_stmt);
+      else
+	return false;
+      if (vce_stmt)
+	{
+	  tree vce_rhs = gimple_assign_rhs1 (vce_stmt);
+	  if (TREE_CODE (vce_rhs) != VIEW_CONVERT_EXPR
+	      || !operand_equal_p (TREE_OPERAND (vce_rhs, 0), loaded_val))
+	    return false;
+	  if (!INTEGRAL_TYPE_P (TREE_TYPE (vce_rhs))
+	      || !SCALAR_FLOAT_TYPE_P (TREE_TYPE (loaded_val))
+	      || !tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vce_rhs)),
+				      TYPE_SIZE (TREE_TYPE (loaded_val))))
+	    return false;
+	  gsi_prev_nondebug (&gsi);
+	  if (!gsi_end_p (gsi))
+	    return false;
+	}
+    }
+  tree cond = gimple_assign_rhs1 (condexpr_stmt);
+  tree cond_op1, cond_op2;
+  if (cond_stmt)
+    {
+      if (!operand_equal_p (cond, gimple_assign_lhs (cond_stmt)))
+	return false;
+      cond_op1 = gimple_assign_rhs1 (cond_stmt);
+      cond_op2 = gimple_assign_rhs2 (cond_stmt);
+    }
+  else if (TREE_CODE (cond) != EQ_EXPR && TREE_CODE (cond) != NE_EXPR)
+    return false;
+  else
+    {
+      cond_op1 = TREE_OPERAND (cond, 0);
+      cond_op2 = TREE_OPERAND (cond, 1);
+    }
+  tree d;
+  if (TREE_CODE (cond) == NE_EXPR)
+    {
+      if (!operand_equal_p (gimple_assign_rhs2 (condexpr_stmt), loaded_val))
+	return false;
+      d = gimple_assign_rhs3 (condexpr_stmt);
+    }
+  else if (!operand_equal_p (gimple_assign_rhs3 (condexpr_stmt), loaded_val))
+    return false;
+  else
+    d = gimple_assign_rhs2 (condexpr_stmt);
+  tree e = vce_stmt ? gimple_assign_lhs (vce_stmt) : loaded_val;
+  if (operand_equal_p (e, cond_op1))
+    e = cond_op2;
+  else if (operand_equal_p (e, cond_op2))
+    e = cond_op1;
+  else
+    return false;
+
+  location_t loc = gimple_location (store_stmt);
+  gimple *load_stmt = last_stmt (load_bb);
+  bool need_new = gimple_omp_atomic_need_value_p (store_stmt);
+  bool need_old = gimple_omp_atomic_need_value_p (load_stmt);
+  bool weak = gimple_omp_atomic_weak_p (load_stmt);
+  enum omp_memory_order omo = gimple_omp_atomic_memory_order (load_stmt);
+  tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
+  tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo));
+  gcc_checking_assert (!need_old || !need_new);
+
+  enum built_in_function fncode
+    = (enum built_in_function) ((int) BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
+				+ index + 1);
+  tree cmpxchg = builtin_decl_explicit (fncode);
+  if (cmpxchg == NULL_TREE)
+    return false;
+  tree itype = TREE_TYPE (TREE_TYPE (cmpxchg));
+
+  if (!can_compare_and_swap_p (TYPE_MODE (itype), true)
+      || !can_atomic_load_p (TYPE_MODE (itype)))
+    return false;
+
+  tree type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val));
+  if (SCALAR_FLOAT_TYPE_P (type) && !vce_stmt)
+    return false;
+
+  gsi = gsi_for_stmt (store_stmt);
+  if (!useless_type_conversion_p (itype, TREE_TYPE (e)))
+    {
+      tree ne = create_tmp_reg (itype);
+      gimple *g = gimple_build_assign (ne, NOP_EXPR, e);
+      gimple_set_location (g, loc);
+      gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+      e = ne;
+    }
+  if (!useless_type_conversion_p (itype, TREE_TYPE (d)))
+    {
+      tree nd = create_tmp_reg (itype);
+      enum tree_code code;
+      if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (d)))
+	{
+	  code = VIEW_CONVERT_EXPR;
+	  d = build1 (VIEW_CONVERT_EXPR, itype, d);
+	}
+      else
+	code = NOP_EXPR;
+      gimple *g = gimple_build_assign (nd, code, d);
+      gimple_set_location (g, loc);
+      gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+      d = nd;
+    }
+
+  tree ctype = build_complex_type (itype);
+  int flag = int_size_in_bytes (itype) + (weak ? 256 : 0);
+  gimple *g
+    = gimple_build_call_internal (IFN_ATOMIC_COMPARE_EXCHANGE, 6, addr, e, d,
+				  build_int_cst (integer_type_node, flag),
+				  mo, fmo);
+  tree cres = create_tmp_reg (ctype);
+  gimple_call_set_lhs (g, cres);
+  gimple_set_location (g, loc);
+  gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+
+  if (cond_stmt || need_old || need_new)
+    {
+      tree im = create_tmp_reg (itype);
+      g = gimple_build_assign (im, IMAGPART_EXPR,
+			       build1 (IMAGPART_EXPR, itype, cres));
+      gimple_set_location (g, loc);
+      gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+
+      tree re = NULL_TREE;
+      if (need_old || need_new)
+	{
+	  re = create_tmp_reg (itype);
+	  g = gimple_build_assign (re, REALPART_EXPR,
+				   build1 (REALPART_EXPR, itype, cres));
+	  gimple_set_location (g, loc);
+	  gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+	}
+
+      if (cond_stmt)
+	{
+	  g = gimple_build_assign (gimple_assign_lhs (cond_stmt),
+				   NOP_EXPR, im);
+	  gimple_set_location (g, loc);
+	  gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+	}
+      else if (need_new)
+	{
+	  g = gimple_build_assign (create_tmp_reg (itype), COND_EXPR,
+				   build2 (NE_EXPR, boolean_type_node,
+					   im, build_zero_cst (itype)),
+				   d, re);
+	  gimple_set_location (g, loc);
+	  gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+	  re = gimple_assign_lhs (g);
+	}
+
+      if (need_old || need_new)
+	{
+	  tree v = need_old ? loaded_val : stored_val;
+	  enum tree_code code;
+	  if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (v)))
+	    {
+	      code = VIEW_CONVERT_EXPR;
+	      re = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (v), re);
+	    }
+	  else if (!useless_type_conversion_p (TREE_TYPE (v), itype))
+	    code = NOP_EXPR;
+	  else
+	    code = TREE_CODE (re);
+	  g = gimple_build_assign (v, code, re);
+	  gimple_set_location (g, loc);
+	  gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+	}
+    }
+
+  gsi_remove (&gsi, true);
+  gsi = gsi_for_stmt (load_stmt);
+  gsi_remove (&gsi, true);
+  gsi = gsi_for_stmt (condexpr_stmt);
+  gsi_remove (&gsi, true);
+  if (cond_stmt)
+    {
+      gsi = gsi_for_stmt (cond_stmt);
+      gsi_remove (&gsi, true);
+    }
+  if (vce_stmt)
+    {
+      gsi = gsi_for_stmt (vce_stmt);
+      gsi_remove (&gsi, true);
+    }
+
+  return true;
+}
+
 /* A subroutine of expand_omp_atomic.  Implement the atomic operation as:
 
       oldval = *addr;
@@ -8825,13 +9116,8 @@ expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
   gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
   location_t loc = gimple_location (gsi_stmt (si));
   enum omp_memory_order omo = gimple_omp_atomic_memory_order (gsi_stmt (si));
-  enum memmodel imo = omp_memory_order_to_memmodel (omo);
-  tree mo = build_int_cst (NULL, imo);
-  if (imo == MEMMODEL_RELEASE)
-    imo = MEMMODEL_RELAXED;
-  else if (imo == MEMMODEL_ACQ_REL)
-    imo = MEMMODEL_ACQUIRE;
-  tree fmo = build_int_cst (NULL, imo);
+  tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
+  tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo));
 
   /* For floating-point values, we'll need to view-convert them to integers
      so that we can perform the atomic compare and swap.  Simplify the
@@ -9114,6 +9400,13 @@ expand_omp_atomic (struct omp_region *region)
 					     loaded_val, stored_val, index))
 	    return;
 
+	  /* When possible, use ATOMIC_COMPARE_EXCHANGE ifn without a loop.  */
+	  if (store_bb == single_succ (load_bb)
+	      && !gimple_in_ssa_p (cfun)
+	      && expand_omp_atomic_cas (load_bb, addr, loaded_val, stored_val,
+					index))
+	    return;
+
 	  /* If we don't have specialized __sync builtins, try and implement
 	     as a compare and swap loop.  */
 	  if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,