diff options
Diffstat (limited to 'gcc/omp-expand.c')
| -rw-r--r-- | gcc/omp-expand.c | 323 |
1 files changed, 308 insertions, 15 deletions
diff --git a/gcc/omp-expand.c b/gcc/omp-expand.c index f2b6f6f..159ae0e 100644 --- a/gcc/omp-expand.c +++ b/gcc/omp-expand.c @@ -8487,22 +8487,58 @@ expand_omp_synch (struct omp_region *region) } } +/* Translate enum omp_memory_order to enum memmodel for the embedded + fail clause in there. */ + +static enum memmodel +omp_memory_order_to_fail_memmodel (enum omp_memory_order mo) +{ + switch (mo & OMP_FAIL_MEMORY_ORDER_MASK) + { + case OMP_FAIL_MEMORY_ORDER_UNSPECIFIED: + switch (mo & OMP_MEMORY_ORDER_MASK) + { + case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED; + case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE; + case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELAXED; + case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQUIRE; + case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST; + default: break; + } + gcc_unreachable (); + case OMP_FAIL_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED; + case OMP_FAIL_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE; + case OMP_FAIL_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST; + default: gcc_unreachable (); + } +} + /* Translate enum omp_memory_order to enum memmodel. The two enums are using different numbers so that OMP_MEMORY_ORDER_UNSPECIFIED - is 0. */ + is 0 and omp_memory_order has the fail mode encoded in it too. */ static enum memmodel omp_memory_order_to_memmodel (enum omp_memory_order mo) { - switch (mo) - { - case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED; - case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE; - case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELEASE; - case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQ_REL; - case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST; + enum memmodel ret, fail_ret; + switch (mo & OMP_MEMORY_ORDER_MASK) + { + case OMP_MEMORY_ORDER_RELAXED: ret = MEMMODEL_RELAXED; break; + case OMP_MEMORY_ORDER_ACQUIRE: ret = MEMMODEL_ACQUIRE; break; + case OMP_MEMORY_ORDER_RELEASE: ret = MEMMODEL_RELEASE; break; + case OMP_MEMORY_ORDER_ACQ_REL: ret = MEMMODEL_ACQ_REL; break; + case OMP_MEMORY_ORDER_SEQ_CST: ret = MEMMODEL_SEQ_CST; break; default: gcc_unreachable (); } + /* If we drop the -Winvalid-memory-model warning for C++17 P0418R2, + we can just return ret here unconditionally. Otherwise, work around + it here and make sure fail memmodel is not stronger. */ + if ((mo & OMP_FAIL_MEMORY_ORDER_MASK) == OMP_FAIL_MEMORY_ORDER_UNSPECIFIED) + return ret; + fail_ret = omp_memory_order_to_fail_memmodel (mo); + if (fail_ret > ret) + return fail_ret; + return ret; } /* A subroutine of expand_omp_atomic. Attempt to implement the atomic @@ -8782,6 +8818,261 @@ expand_omp_atomic_fetch_op (basic_block load_bb, return true; } +/* A subroutine of expand_omp_atomic. Attempt to implement the atomic + compare and exchange as an ATOMIC_COMPARE_EXCHANGE internal function. + Returns false if the expression is not of the proper form. */ + +static bool +expand_omp_atomic_cas (basic_block load_bb, tree addr, + tree loaded_val, tree stored_val, int index) +{ + /* We expect to find the following sequences: + + load_bb: + GIMPLE_OMP_ATOMIC_LOAD (tmp, mem) + + store_bb: + val = tmp == e ? d : tmp; + GIMPLE_OMP_ATOMIC_STORE (val) + + or in store_bb instead: + tmp2 = tmp == e; + val = tmp2 ? d : tmp; + GIMPLE_OMP_ATOMIC_STORE (val) + + or: + tmp3 = VIEW_CONVERT_EXPR<integral_type>(tmp); + val = e == tmp3 ? d : tmp; + GIMPLE_OMP_ATOMIC_STORE (val) + + etc. */ + + + basic_block store_bb = single_succ (load_bb); + gimple_stmt_iterator gsi = gsi_last_nondebug_bb (store_bb); + gimple *store_stmt = gsi_stmt (gsi); + if (!store_stmt || gimple_code (store_stmt) != GIMPLE_OMP_ATOMIC_STORE) + return false; + gsi_prev_nondebug (&gsi); + if (gsi_end_p (gsi)) + return false; + gimple *condexpr_stmt = gsi_stmt (gsi); + if (!is_gimple_assign (condexpr_stmt) + || gimple_assign_rhs_code (condexpr_stmt) != COND_EXPR) + return false; + if (!operand_equal_p (gimple_assign_lhs (condexpr_stmt), stored_val, 0)) + return false; + gimple *cond_stmt = NULL; + gimple *vce_stmt = NULL; + gsi_prev_nondebug (&gsi); + if (!gsi_end_p (gsi)) + { + cond_stmt = gsi_stmt (gsi); + if (!is_gimple_assign (cond_stmt)) + return false; + if (gimple_assign_rhs_code (cond_stmt) == EQ_EXPR) + { + gsi_prev_nondebug (&gsi); + if (!gsi_end_p (gsi)) + { + vce_stmt = gsi_stmt (gsi); + if (!is_gimple_assign (vce_stmt) + || gimple_assign_rhs_code (vce_stmt) != VIEW_CONVERT_EXPR) + return false; + } + } + else if (gimple_assign_rhs_code (cond_stmt) == VIEW_CONVERT_EXPR) + std::swap (vce_stmt, cond_stmt); + else + return false; + if (vce_stmt) + { + tree vce_rhs = gimple_assign_rhs1 (vce_stmt); + if (TREE_CODE (vce_rhs) != VIEW_CONVERT_EXPR + || !operand_equal_p (TREE_OPERAND (vce_rhs, 0), loaded_val)) + return false; + if (!INTEGRAL_TYPE_P (TREE_TYPE (vce_rhs)) + || !SCALAR_FLOAT_TYPE_P (TREE_TYPE (loaded_val)) + || !tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vce_rhs)), + TYPE_SIZE (TREE_TYPE (loaded_val)))) + return false; + gsi_prev_nondebug (&gsi); + if (!gsi_end_p (gsi)) + return false; + } + } + tree cond = gimple_assign_rhs1 (condexpr_stmt); + tree cond_op1, cond_op2; + if (cond_stmt) + { + if (!operand_equal_p (cond, gimple_assign_lhs (cond_stmt))) + return false; + cond_op1 = gimple_assign_rhs1 (cond_stmt); + cond_op2 = gimple_assign_rhs2 (cond_stmt); + } + else if (TREE_CODE (cond) != EQ_EXPR && TREE_CODE (cond) != NE_EXPR) + return false; + else + { + cond_op1 = TREE_OPERAND (cond, 0); + cond_op2 = TREE_OPERAND (cond, 1); + } + tree d; + if (TREE_CODE (cond) == NE_EXPR) + { + if (!operand_equal_p (gimple_assign_rhs2 (condexpr_stmt), loaded_val)) + return false; + d = gimple_assign_rhs3 (condexpr_stmt); + } + else if (!operand_equal_p (gimple_assign_rhs3 (condexpr_stmt), loaded_val)) + return false; + else + d = gimple_assign_rhs2 (condexpr_stmt); + tree e = vce_stmt ? gimple_assign_lhs (vce_stmt) : loaded_val; + if (operand_equal_p (e, cond_op1)) + e = cond_op2; + else if (operand_equal_p (e, cond_op2)) + e = cond_op1; + else + return false; + + location_t loc = gimple_location (store_stmt); + gimple *load_stmt = last_stmt (load_bb); + bool need_new = gimple_omp_atomic_need_value_p (store_stmt); + bool need_old = gimple_omp_atomic_need_value_p (load_stmt); + bool weak = gimple_omp_atomic_weak_p (load_stmt); + enum omp_memory_order omo = gimple_omp_atomic_memory_order (load_stmt); + tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo)); + tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo)); + gcc_checking_assert (!need_old || !need_new); + + enum built_in_function fncode + = (enum built_in_function) ((int) BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N + + index + 1); + tree cmpxchg = builtin_decl_explicit (fncode); + if (cmpxchg == NULL_TREE) + return false; + tree itype = TREE_TYPE (TREE_TYPE (cmpxchg)); + + if (!can_compare_and_swap_p (TYPE_MODE (itype), true) + || !can_atomic_load_p (TYPE_MODE (itype))) + return false; + + tree type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val)); + if (SCALAR_FLOAT_TYPE_P (type) && !vce_stmt) + return false; + + gsi = gsi_for_stmt (store_stmt); + if (!useless_type_conversion_p (itype, TREE_TYPE (e))) + { + tree ne = create_tmp_reg (itype); + gimple *g = gimple_build_assign (ne, NOP_EXPR, e); + gimple_set_location (g, loc); + gsi_insert_before (&gsi, g, GSI_SAME_STMT); + e = ne; + } + if (!useless_type_conversion_p (itype, TREE_TYPE (d))) + { + tree nd = create_tmp_reg (itype); + enum tree_code code; + if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (d))) + { + code = VIEW_CONVERT_EXPR; + d = build1 (VIEW_CONVERT_EXPR, itype, d); + } + else + code = NOP_EXPR; + gimple *g = gimple_build_assign (nd, code, d); + gimple_set_location (g, loc); + gsi_insert_before (&gsi, g, GSI_SAME_STMT); + d = nd; + } + + tree ctype = build_complex_type (itype); + int flag = int_size_in_bytes (itype) + (weak ? 256 : 0); + gimple *g + = gimple_build_call_internal (IFN_ATOMIC_COMPARE_EXCHANGE, 6, addr, e, d, + build_int_cst (integer_type_node, flag), + mo, fmo); + tree cres = create_tmp_reg (ctype); + gimple_call_set_lhs (g, cres); + gimple_set_location (g, loc); + gsi_insert_before (&gsi, g, GSI_SAME_STMT); + + if (cond_stmt || need_old || need_new) + { + tree im = create_tmp_reg (itype); + g = gimple_build_assign (im, IMAGPART_EXPR, + build1 (IMAGPART_EXPR, itype, cres)); + gimple_set_location (g, loc); + gsi_insert_before (&gsi, g, GSI_SAME_STMT); + + tree re = NULL_TREE; + if (need_old || need_new) + { + re = create_tmp_reg (itype); + g = gimple_build_assign (re, REALPART_EXPR, + build1 (REALPART_EXPR, itype, cres)); + gimple_set_location (g, loc); + gsi_insert_before (&gsi, g, GSI_SAME_STMT); + } + + if (cond_stmt) + { + g = gimple_build_assign (gimple_assign_lhs (cond_stmt), + NOP_EXPR, im); + gimple_set_location (g, loc); + gsi_insert_before (&gsi, g, GSI_SAME_STMT); + } + else if (need_new) + { + g = gimple_build_assign (create_tmp_reg (itype), COND_EXPR, + build2 (NE_EXPR, boolean_type_node, + im, build_zero_cst (itype)), + d, re); + gimple_set_location (g, loc); + gsi_insert_before (&gsi, g, GSI_SAME_STMT); + re = gimple_assign_lhs (g); + } + + if (need_old || need_new) + { + tree v = need_old ? loaded_val : stored_val; + enum tree_code code; + if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (v))) + { + code = VIEW_CONVERT_EXPR; + re = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (v), re); + } + else if (!useless_type_conversion_p (TREE_TYPE (v), itype)) + code = NOP_EXPR; + else + code = TREE_CODE (re); + g = gimple_build_assign (v, code, re); + gimple_set_location (g, loc); + gsi_insert_before (&gsi, g, GSI_SAME_STMT); + } + } + + gsi_remove (&gsi, true); + gsi = gsi_for_stmt (load_stmt); + gsi_remove (&gsi, true); + gsi = gsi_for_stmt (condexpr_stmt); + gsi_remove (&gsi, true); + if (cond_stmt) + { + gsi = gsi_for_stmt (cond_stmt); + gsi_remove (&gsi, true); + } + if (vce_stmt) + { + gsi = gsi_for_stmt (vce_stmt); + gsi_remove (&gsi, true); + } + + return true; +} + /* A subroutine of expand_omp_atomic. Implement the atomic operation as: oldval = *addr; @@ -8825,13 +9116,8 @@ expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb, gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD); location_t loc = gimple_location (gsi_stmt (si)); enum omp_memory_order omo = gimple_omp_atomic_memory_order (gsi_stmt (si)); - enum memmodel imo = omp_memory_order_to_memmodel (omo); - tree mo = build_int_cst (NULL, imo); - if (imo == MEMMODEL_RELEASE) - imo = MEMMODEL_RELAXED; - else if (imo == MEMMODEL_ACQ_REL) - imo = MEMMODEL_ACQUIRE; - tree fmo = build_int_cst (NULL, imo); + tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo)); + tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo)); /* For floating-point values, we'll need to view-convert them to integers so that we can perform the atomic compare and swap. Simplify the @@ -9114,6 +9400,13 @@ expand_omp_atomic (struct omp_region *region) loaded_val, stored_val, index)) return; + /* When possible, use ATOMIC_COMPARE_EXCHANGE ifn without a loop. */ + if (store_bb == single_succ (load_bb) + && !gimple_in_ssa_p (cfun) + && expand_omp_atomic_cas (load_bb, addr, loaded_val, stored_val, + index)) + return; + /* If we don't have specialized __sync builtins, try and implement as a compare and swap loop. */ if (expand_omp_atomic_pipeline (load_bb, store_bb, addr, |