From 8ebedfcd86aa5e3fc902fb442ce12c9d440c23c8 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Fri, 6 Nov 2020 09:35:27 +0100 Subject: tree-optimization/97732 - fix init of SLP induction vectorization This PR exposes two issues - one that the vector builder treats &x as eligible for VECTOR_CST elements and one that SLP induction vectorization forgets to convert init elements to the vector component type which makes a difference for pointer vs. integer. 2020-11-06 Richard Biener PR tree-optimization/97732 * tree-vect-loop.c (vectorizable_induction): Convert the init elements to the vector component type. * gimple-fold.c (gimple_build_vector): Use CONSTANT_CLASS_P rather than TREE_CONSTANT to determine if elements are eligible for VECTOR_CSTs. * gcc.dg/vect/bb-slp-pr97732.c: New testcase. --- gcc/gimple-fold.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index c3fa4cb..ca38a31 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -7855,7 +7855,7 @@ gimple_build_vector (gimple_seq *seq, location_t loc, gcc_assert (builder->nelts_per_pattern () <= 2); unsigned int encoded_nelts = builder->encoded_nelts (); for (unsigned int i = 0; i < encoded_nelts; ++i) - if (!TREE_CONSTANT ((*builder)[i])) + if (!CONSTANT_CLASS_P ((*builder)[i])) { tree type = builder->type (); unsigned int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant (); -- cgit v1.1 From 1bea0d0aa5936cb36b6f86f721ca03c1a1bb601d Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Fri, 20 Nov 2020 12:28:34 +0100 Subject: c++: Add __builtin_clear_padding builtin - C++20 P0528R3 compiler side [PR88101] The following patch implements __builtin_clear_padding builtin that clears the padding bits in object representation (but preserves value representation). Inside of unions it clears only those padding bits that are padding for all the union members (so that it never alters value representation). It handles trailing padding, padding in the middle of structs including bitfields (PDP11 unhandled, I've never figured out how those bitfields work), VLAs (doesn't handle variable length structures, but I think almost nobody uses them and it isn't worth the extra complexity). For VLAs and sufficiently large arrays it uses runtime clearing loop instead of emitting straight-line code (unless arrays are inside of a union). The way I think this can be used for atomics is e.g. if the structures are power of two sized and small enough that we use the hw atomics for say compare_exchange __builtin_clear_padding could be called first on the address of expected and desired arguments (for desired only if we want to ensure that most of the time the atomic memory will have padding bits cleared), then perform the weak cmpxchg and if that fails, we got the value from the atomic memory; we can call __builtin_clear_padding on a copy of that and then compare it with expected, and if it is the same with the padding bits masked off, we can use the original with whatever random padding bits in it as the new expected for next cmpxchg. __builtin_clear_padding itself is not atomic and therefore it shouldn't be called on the atomic memory itself, but compare_exchange*'s expected argument is a reference and normally the implementation may store there the current value from memory, so padding bits can be cleared in that, and desired is passed by value rather than reference, so clearing is fine too. When using libatomic, we can use it either that way, or add new libatomic APIs that accept another argument, pointer to the padding bit bitmask, and construct that in the template as alignas (_T) unsigned char _mask[sizeof (_T)]; std::memset (_mask, ~0, sizeof (_mask)); __builtin_clear_padding ((_T *) _mask); which will have bits cleared for padding bits and set for bits taking part in the value representation. Then libatomic could internally instead of using memcmp compare for (i = 0; i < N; i++) if ((val1[i] & mask[i]) != (val2[i] & mask[i])) 2020-11-20 Jakub Jelinek PR libstdc++/88101 gcc/ * builtins.def (BUILT_IN_CLEAR_PADDING): New built-in function. * gimplify.c (gimplify_call_expr): Rewrite single argument BUILT_IN_CLEAR_PADDING into two-argument variant. * gimple-fold.c (clear_padding_unit, clear_padding_buf_size): New const variables. (struct clear_padding_struct): New type. (clear_padding_flush, clear_padding_add_padding, clear_padding_emit_loop, clear_padding_type, clear_padding_union, clear_padding_real_needs_padding_p, clear_padding_type_may_have_padding_p, gimple_fold_builtin_clear_padding): New functions. (gimple_fold_builtin): Handle BUILT_IN_CLEAR_PADDING. * doc/extend.texi (__builtin_clear_padding): Document. gcc/c-family/ * c-common.c (check_builtin_function_arguments): Handle BUILT_IN_CLEAR_PADDING. gcc/testsuite/ * c-c++-common/builtin-clear-padding-1.c: New test. * c-c++-common/torture/builtin-clear-padding-1.c: New test. * c-c++-common/torture/builtin-clear-padding-2.c: New test. * c-c++-common/torture/builtin-clear-padding-3.c: New test. * c-c++-common/torture/builtin-clear-padding-4.c: New test. * c-c++-common/torture/builtin-clear-padding-5.c: New test. * g++.dg/torture/builtin-clear-padding-1.C: New test. * g++.dg/torture/builtin-clear-padding-2.C: New test. * gcc.dg/builtin-clear-padding-1.c: New test. --- gcc/gimple-fold.c | 695 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 695 insertions(+) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index ca38a31..5d60bef 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -3948,6 +3948,698 @@ gimple_fold_builtin_realloc (gimple_stmt_iterator *gsi) return false; } +/* Number of bytes into which any type but aggregate or vector types + should fit. */ +static constexpr size_t clear_padding_unit + = MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT; +/* Buffer size on which __builtin_clear_padding folding code works. */ +static const size_t clear_padding_buf_size = 32 * clear_padding_unit; + +/* Data passed through __builtin_clear_padding folding. */ +struct clear_padding_struct { + location_t loc; + tree base; + tree alias_type; + gimple_stmt_iterator *gsi; + /* Alignment of buf->base + 0. */ + unsigned align; + /* Offset from buf->base. Should be always a multiple of UNITS_PER_WORD. */ + HOST_WIDE_INT off; + /* Number of padding bytes before buf->off that don't have padding clear + code emitted yet. */ + HOST_WIDE_INT padding_bytes; + /* The size of the whole object. Never emit code to touch + buf->base + buf->sz or following bytes. */ + HOST_WIDE_INT sz; + /* Number of bytes recorded in buf->buf. */ + size_t size; + /* When inside union, instead of emitting code we and bits inside of + the union_ptr array. */ + unsigned char *union_ptr; + /* Set bits mean padding bits that need to be cleared by the builtin. */ + unsigned char buf[clear_padding_buf_size + clear_padding_unit]; +}; + +/* Emit code to clear padding requested in BUF->buf - set bits + in there stand for padding that should be cleared. FULL is true + if everything from the buffer should be flushed, otherwise + it can leave up to 2 * clear_padding_unit bytes for further + processing. */ + +static void +clear_padding_flush (clear_padding_struct *buf, bool full) +{ + gcc_assert ((clear_padding_unit % UNITS_PER_WORD) == 0); + if (!full && buf->size < 2 * clear_padding_unit) + return; + gcc_assert ((buf->off % UNITS_PER_WORD) == 0); + size_t end = buf->size; + if (!full) + end = ((end - clear_padding_unit - 1) / clear_padding_unit + * clear_padding_unit); + size_t padding_bytes = buf->padding_bytes; + if (buf->union_ptr) + { + /* Inside of a union, instead of emitting any code, instead + clear all bits in the union_ptr buffer that are clear + in buf. Whole padding bytes don't clear anything. */ + for (size_t i = 0; i < end; i++) + { + if (buf->buf[i] == (unsigned char) ~0) + padding_bytes++; + else + { + padding_bytes = 0; + buf->union_ptr[buf->off + i] &= buf->buf[i]; + } + } + if (full) + { + buf->off = 0; + buf->size = 0; + buf->padding_bytes = 0; + } + else + { + memmove (buf->buf, buf->buf + end, buf->size - end); + buf->off += end; + buf->size -= end; + buf->padding_bytes = padding_bytes; + } + return; + } + size_t wordsize = UNITS_PER_WORD; + for (size_t i = 0; i < end; i += wordsize) + { + size_t nonzero_first = wordsize; + size_t nonzero_last = 0; + bool all_ones = true; + if ((unsigned HOST_WIDE_INT) (buf->off + i + wordsize) + > (unsigned HOST_WIDE_INT) buf->sz) + { + gcc_assert (wordsize > 1); + wordsize /= 2; + i -= wordsize; + continue; + } + for (size_t j = i; j < i + wordsize && j < end; j++) + { + if (buf->buf[j]) + { + if (nonzero_first == wordsize) + { + nonzero_first = j - i; + nonzero_last = j - i; + } + if (nonzero_last != j - i) + all_ones = false; + nonzero_last = j + 1 - i; + } + if (buf->buf[j] != 0 && buf->buf[j] != (unsigned char) ~0) + all_ones = false; + } + if (padding_bytes) + { + if (nonzero_first == 0 + && nonzero_last == wordsize + && all_ones) + { + /* All bits are padding and we had some padding + before too. Just extend it. */ + padding_bytes += wordsize; + continue; + } + size_t padding_end = i; + if (all_ones && nonzero_first == 0) + { + padding_bytes += nonzero_last; + padding_end += nonzero_last; + nonzero_first = wordsize; + nonzero_last = 0; + } + tree atype = build_array_type_nelts (char_type_node, padding_bytes); + tree dst = build2_loc (buf->loc, MEM_REF, atype, buf->base, + build_int_cst (buf->alias_type, + buf->off + padding_end + - padding_bytes)); + tree src = build_constructor (atype, NULL); + gimple *g = gimple_build_assign (dst, src); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + padding_bytes = 0; + buf->padding_bytes = 0; + } + if (nonzero_first == wordsize) + /* All bits in a word are 0, there are no padding bits. */ + continue; + if (all_ones && nonzero_last == wordsize) + { + /* All bits between nonzero_first and end of word are padding + bits, start counting padding_bytes. */ + padding_bytes = nonzero_last - nonzero_first; + continue; + } + for (size_t eltsz = 1; eltsz <= wordsize; eltsz <<= 1) + { + if (nonzero_last - nonzero_first <= eltsz + && ((nonzero_first & ~(eltsz - 1)) + == ((nonzero_last - 1) & ~(eltsz - 1)))) + { + tree type; + if (eltsz == 1) + type = char_type_node; + else + type = lang_hooks.types.type_for_size (eltsz * BITS_PER_UNIT, + 0); + size_t start = nonzero_first & ~(eltsz - 1); + HOST_WIDE_INT off = buf->off + i + start; + tree atype = type; + if (eltsz > 1 && buf->align < TYPE_ALIGN (type)) + atype = build_aligned_type (type, buf->align); + tree dst = build2_loc (buf->loc, MEM_REF, atype, buf->base, + build_int_cst (buf->alias_type, off)); + tree src; + gimple *g; + if (all_ones + && nonzero_first == start + && nonzero_last == start + eltsz) + src = build_zero_cst (type); + else + { + src = make_ssa_name (type); + g = gimple_build_assign (src, unshare_expr (dst)); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + tree mask = native_interpret_expr (type, + buf->buf + i + start, + eltsz); + gcc_assert (mask && TREE_CODE (mask) == INTEGER_CST); + mask = fold_build1 (BIT_NOT_EXPR, type, mask); + tree src_masked = make_ssa_name (type); + g = gimple_build_assign (src_masked, BIT_AND_EXPR, + src, mask); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + src = src_masked; + } + g = gimple_build_assign (dst, src); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + break; + } + } + } + if (full) + { + if (padding_bytes) + { + tree atype = build_array_type_nelts (char_type_node, padding_bytes); + tree dst = build2_loc (buf->loc, MEM_REF, atype, buf->base, + build_int_cst (buf->alias_type, + buf->off + end + - padding_bytes)); + tree src = build_constructor (atype, NULL); + gimple *g = gimple_build_assign (dst, src); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + } + size_t end_rem = end % UNITS_PER_WORD; + buf->off += end - end_rem; + buf->size = end_rem; + memset (buf->buf, 0, buf->size); + buf->padding_bytes = 0; + } + else + { + memmove (buf->buf, buf->buf + end, buf->size - end); + buf->off += end; + buf->size -= end; + buf->padding_bytes = padding_bytes; + } +} + +/* Append PADDING_BYTES padding bytes. */ + +static void +clear_padding_add_padding (clear_padding_struct *buf, + HOST_WIDE_INT padding_bytes) +{ + if (padding_bytes == 0) + return; + if ((unsigned HOST_WIDE_INT) padding_bytes + buf->size + > (unsigned HOST_WIDE_INT) clear_padding_buf_size) + clear_padding_flush (buf, false); + if ((unsigned HOST_WIDE_INT) padding_bytes + buf->size + > (unsigned HOST_WIDE_INT) clear_padding_buf_size) + { + memset (buf->buf + buf->size, ~0, clear_padding_buf_size - buf->size); + padding_bytes -= clear_padding_buf_size - buf->size; + buf->size = clear_padding_buf_size; + clear_padding_flush (buf, false); + gcc_assert (buf->padding_bytes); + /* At this point buf->buf[0] through buf->buf[buf->size - 1] + is guaranteed to be all ones. */ + padding_bytes += buf->size; + buf->size = padding_bytes % UNITS_PER_WORD; + memset (buf->buf, ~0, buf->size); + buf->off += padding_bytes - buf->size; + buf->padding_bytes += padding_bytes - buf->size; + } + else + { + memset (buf->buf + buf->size, ~0, padding_bytes); + buf->size += padding_bytes; + } +} + +static void clear_padding_type (clear_padding_struct *, tree, HOST_WIDE_INT); + +/* Clear padding bits of union type TYPE. */ + +static void +clear_padding_union (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) +{ + clear_padding_struct *union_buf; + HOST_WIDE_INT start_off = 0, next_off = 0; + size_t start_size = 0; + if (buf->union_ptr) + { + start_off = buf->off + buf->size; + next_off = start_off + sz; + start_size = start_off % UNITS_PER_WORD; + start_off -= start_size; + clear_padding_flush (buf, true); + union_buf = buf; + } + else + { + if (sz + buf->size > clear_padding_buf_size) + clear_padding_flush (buf, false); + union_buf = XALLOCA (clear_padding_struct); + union_buf->loc = buf->loc; + union_buf->base = NULL_TREE; + union_buf->alias_type = NULL_TREE; + union_buf->gsi = NULL; + union_buf->align = 0; + union_buf->off = 0; + union_buf->padding_bytes = 0; + union_buf->sz = sz; + union_buf->size = 0; + if (sz + buf->size <= clear_padding_buf_size) + union_buf->union_ptr = buf->buf + buf->size; + else + union_buf->union_ptr = XNEWVEC (unsigned char, sz); + memset (union_buf->union_ptr, ~0, sz); + } + + for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) + if (TREE_CODE (field) == FIELD_DECL) + { + HOST_WIDE_INT fldsz = tree_to_shwi (DECL_SIZE_UNIT (field)); + gcc_assert (union_buf->size == 0); + union_buf->off = start_off; + union_buf->size = start_size; + memset (union_buf->buf, ~0, start_size); + clear_padding_type (union_buf, TREE_TYPE (field), fldsz); + clear_padding_add_padding (union_buf, sz - fldsz); + clear_padding_flush (union_buf, true); + } + + if (buf == union_buf) + { + buf->off = next_off; + buf->size = next_off % UNITS_PER_WORD; + buf->off -= buf->size; + memset (buf->buf, ~0, buf->size); + } + else if (sz + buf->size <= clear_padding_buf_size) + buf->size += sz; + else + { + unsigned char *union_ptr = union_buf->union_ptr; + while (sz) + { + clear_padding_flush (buf, false); + HOST_WIDE_INT this_sz + = MIN ((unsigned HOST_WIDE_INT) sz, + clear_padding_buf_size - buf->size); + memcpy (buf->buf + buf->size, union_ptr, this_sz); + buf->size += this_sz; + union_ptr += this_sz; + sz -= this_sz; + } + XDELETE (union_buf->union_ptr); + } +} + +/* The only known floating point formats with padding bits are the + IEEE extended ones. */ + +static bool +clear_padding_real_needs_padding_p (tree type) +{ + const struct real_format *fmt = REAL_MODE_FORMAT (TYPE_MODE (type)); + return (fmt->b == 2 + && fmt->signbit_ro == fmt->signbit_rw + && (fmt->signbit_ro == 79 || fmt->signbit_ro == 95)); +} + +/* Return true if TYPE might contain any padding bits. */ + +static bool +clear_padding_type_may_have_padding_p (tree type) +{ + switch (TREE_CODE (type)) + { + case RECORD_TYPE: + case UNION_TYPE: + return true; + case ARRAY_TYPE: + case COMPLEX_TYPE: + case VECTOR_TYPE: + return clear_padding_type_may_have_padding_p (TREE_TYPE (type)); + case REAL_TYPE: + return clear_padding_real_needs_padding_p (type); + default: + return false; + } +} + +/* Emit a runtime loop: + for (; buf.base != end; buf.base += sz) + __builtin_clear_padding (buf.base); */ + +static void +clear_padding_emit_loop (clear_padding_struct *buf, tree type, tree end) +{ + tree l1 = create_artificial_label (buf->loc); + tree l2 = create_artificial_label (buf->loc); + tree l3 = create_artificial_label (buf->loc); + gimple *g = gimple_build_goto (l2); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + g = gimple_build_label (l1); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + clear_padding_type (buf, type, buf->sz); + clear_padding_flush (buf, true); + g = gimple_build_assign (buf->base, POINTER_PLUS_EXPR, buf->base, + size_int (buf->sz)); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + g = gimple_build_label (l2); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + g = gimple_build_cond (NE_EXPR, buf->base, end, l1, l3); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + g = gimple_build_label (l3); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); +} + +/* Clear padding bits for TYPE. Called recursively from + gimple_fold_builtin_clear_padding. */ + +static void +clear_padding_type (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) +{ + switch (TREE_CODE (type)) + { + case RECORD_TYPE: + HOST_WIDE_INT cur_pos; + cur_pos = 0; + for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) + if (TREE_CODE (field) == FIELD_DECL) + { + if (DECL_BIT_FIELD (field)) + { + if (DECL_NAME (field) == NULL_TREE) + continue; + HOST_WIDE_INT fldsz = TYPE_PRECISION (TREE_TYPE (field)); + if (fldsz == 0) + continue; + HOST_WIDE_INT pos = int_byte_position (field); + HOST_WIDE_INT bpos + = tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field)); + bpos %= BITS_PER_UNIT; + HOST_WIDE_INT end + = ROUND_UP (bpos + fldsz, BITS_PER_UNIT) / BITS_PER_UNIT; + if (pos + end > cur_pos) + { + clear_padding_add_padding (buf, pos + end - cur_pos); + cur_pos = pos + end; + } + gcc_assert (cur_pos > pos + && ((unsigned HOST_WIDE_INT) buf->size + >= (unsigned HOST_WIDE_INT) cur_pos - pos)); + unsigned char *p = buf->buf + buf->size - (cur_pos - pos); + if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN) + sorry_at (buf->loc, "PDP11 bit-field handling unsupported" + " in %qs", "__builtin_clear_padding"); + else if (BYTES_BIG_ENDIAN) + { + /* Big endian. */ + if (bpos + fldsz <= BITS_PER_UNIT) + *p &= ~(((1 << fldsz) - 1) + << (BITS_PER_UNIT - bpos - fldsz)); + else + { + if (bpos) + { + *p &= ~(((1U << BITS_PER_UNIT) - 1) >> bpos); + p++; + fldsz -= BITS_PER_UNIT - bpos; + } + memset (p, 0, fldsz / BITS_PER_UNIT); + p += fldsz / BITS_PER_UNIT; + fldsz %= BITS_PER_UNIT; + if (fldsz) + *p &= ((1U << BITS_PER_UNIT) - 1) >> fldsz; + } + } + else + { + /* Little endian. */ + if (bpos + fldsz <= BITS_PER_UNIT) + *p &= ~(((1 << fldsz) - 1) << bpos); + else + { + if (bpos) + { + *p &= ~(((1 << BITS_PER_UNIT) - 1) << bpos); + p++; + fldsz -= BITS_PER_UNIT - bpos; + } + memset (p, 0, fldsz / BITS_PER_UNIT); + p += fldsz / BITS_PER_UNIT; + fldsz %= BITS_PER_UNIT; + if (fldsz) + *p &= ~((1 << fldsz) - 1); + } + } + } + else + { + HOST_WIDE_INT pos = int_byte_position (field); + HOST_WIDE_INT fldsz = tree_to_shwi (DECL_SIZE_UNIT (field)); + gcc_assert (pos >= 0 && fldsz >= 0 && pos >= cur_pos); + clear_padding_add_padding (buf, pos - cur_pos); + cur_pos = pos; + clear_padding_type (buf, TREE_TYPE (field), fldsz); + cur_pos += fldsz; + } + } + gcc_assert (sz >= cur_pos); + clear_padding_add_padding (buf, sz - cur_pos); + break; + case ARRAY_TYPE: + HOST_WIDE_INT nelts, fldsz; + fldsz = int_size_in_bytes (TREE_TYPE (type)); + nelts = sz / fldsz; + if (nelts > 1 + && sz > 8 * UNITS_PER_WORD + && buf->union_ptr == NULL + && clear_padding_type_may_have_padding_p (TREE_TYPE (type))) + { + /* For sufficiently large array of more than one elements, + emit a runtime loop to keep code size manageable. */ + tree base = buf->base; + unsigned int prev_align = buf->align; + HOST_WIDE_INT off = buf->off + buf->size; + HOST_WIDE_INT prev_sz = buf->sz; + clear_padding_flush (buf, true); + tree elttype = TREE_TYPE (type); + buf->base = create_tmp_var (build_pointer_type (elttype)); + tree end = make_ssa_name (TREE_TYPE (buf->base)); + gimple *g = gimple_build_assign (buf->base, POINTER_PLUS_EXPR, + base, size_int (off)); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + g = gimple_build_assign (end, POINTER_PLUS_EXPR, buf->base, + size_int (sz)); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + buf->sz = fldsz; + buf->align = TYPE_ALIGN (elttype); + buf->off = 0; + buf->size = 0; + clear_padding_emit_loop (buf, elttype, end); + buf->base = base; + buf->sz = prev_sz; + buf->align = prev_align; + buf->size = off % UNITS_PER_WORD; + buf->off = off - buf->size; + memset (buf->buf, 0, buf->size); + break; + } + for (HOST_WIDE_INT i = 0; i < nelts; i++) + clear_padding_type (buf, TREE_TYPE (type), fldsz); + break; + case UNION_TYPE: + clear_padding_union (buf, type, sz); + break; + case REAL_TYPE: + gcc_assert ((size_t) sz <= clear_padding_unit); + if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size) + clear_padding_flush (buf, false); + if (clear_padding_real_needs_padding_p (type)) + { + /* Use native_interpret_expr + native_encode_expr to figure out + which bits are padding. */ + memset (buf->buf + buf->size, ~0, sz); + tree cst = native_interpret_expr (type, buf->buf + buf->size, sz); + gcc_assert (cst && TREE_CODE (cst) == REAL_CST); + int len = native_encode_expr (cst, buf->buf + buf->size, sz); + gcc_assert (len > 0 && (size_t) len == (size_t) sz); + for (size_t i = 0; i < (size_t) sz; i++) + buf->buf[buf->size + i] ^= ~0; + } + else + memset (buf->buf + buf->size, 0, sz); + buf->size += sz; + break; + case COMPLEX_TYPE: + fldsz = int_size_in_bytes (TREE_TYPE (type)); + clear_padding_type (buf, TREE_TYPE (type), fldsz); + clear_padding_type (buf, TREE_TYPE (type), fldsz); + break; + case VECTOR_TYPE: + nelts = TYPE_VECTOR_SUBPARTS (type).to_constant (); + fldsz = int_size_in_bytes (TREE_TYPE (type)); + for (HOST_WIDE_INT i = 0; i < nelts; i++) + clear_padding_type (buf, TREE_TYPE (type), fldsz); + break; + case NULLPTR_TYPE: + gcc_assert ((size_t) sz <= clear_padding_unit); + if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size) + clear_padding_flush (buf, false); + memset (buf->buf + buf->size, ~0, sz); + buf->size += sz; + break; + default: + gcc_assert ((size_t) sz <= clear_padding_unit); + if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size) + clear_padding_flush (buf, false); + memset (buf->buf + buf->size, 0, sz); + buf->size += sz; + break; + } +} + +/* Fold __builtin_clear_padding builtin. */ + +static bool +gimple_fold_builtin_clear_padding (gimple_stmt_iterator *gsi) +{ + gimple *stmt = gsi_stmt (*gsi); + gcc_assert (gimple_call_num_args (stmt) == 2); + tree ptr = gimple_call_arg (stmt, 0); + tree typearg = gimple_call_arg (stmt, 1); + tree type = TREE_TYPE (TREE_TYPE (typearg)); + location_t loc = gimple_location (stmt); + clear_padding_struct buf; + gimple_stmt_iterator gsiprev = *gsi; + /* This should be folded during the lower pass. */ + gcc_assert (!gimple_in_ssa_p (cfun) && cfun->cfg == NULL); + gcc_assert (COMPLETE_TYPE_P (type)); + gsi_prev (&gsiprev); + + buf.loc = loc; + buf.base = ptr; + buf.alias_type = NULL_TREE; + buf.gsi = gsi; + buf.align = get_pointer_alignment (ptr); + unsigned int talign = min_align_of_type (type) * BITS_PER_UNIT; + buf.align = MAX (buf.align, talign); + buf.off = 0; + buf.padding_bytes = 0; + buf.size = 0; + buf.sz = int_size_in_bytes (type); + buf.union_ptr = NULL; + if (buf.sz < 0 && int_size_in_bytes (strip_array_types (type)) < 0) + sorry_at (loc, "%s not supported for variable length aggregates", + "__builtin_clear_padding"); + /* The implementation currently assumes 8-bit host and target + chars which is the case for all currently supported targets + and hosts and is required e.g. for native_{encode,interpret}* APIs. */ + else if (CHAR_BIT != 8 || BITS_PER_UNIT != 8) + sorry_at (loc, "%s not supported on this target", + "__builtin_clear_padding"); + else if (!clear_padding_type_may_have_padding_p (type)) + ; + else if (TREE_CODE (type) == ARRAY_TYPE && buf.sz < 0) + { + tree sz = TYPE_SIZE_UNIT (type); + tree elttype = type; + /* Only supports C/C++ VLAs and flattens all the VLA levels. */ + while (TREE_CODE (elttype) == ARRAY_TYPE + && int_size_in_bytes (elttype) < 0) + elttype = TREE_TYPE (elttype); + HOST_WIDE_INT eltsz = int_size_in_bytes (elttype); + gcc_assert (eltsz >= 0); + if (eltsz) + { + buf.base = create_tmp_var (build_pointer_type (elttype)); + tree end = make_ssa_name (TREE_TYPE (buf.base)); + gimple *g = gimple_build_assign (buf.base, ptr); + gimple_set_location (g, loc); + gsi_insert_before (gsi, g, GSI_SAME_STMT); + g = gimple_build_assign (end, POINTER_PLUS_EXPR, buf.base, sz); + gimple_set_location (g, loc); + gsi_insert_before (gsi, g, GSI_SAME_STMT); + buf.sz = eltsz; + buf.align = TYPE_ALIGN (elttype); + buf.alias_type = build_pointer_type (elttype); + clear_padding_emit_loop (&buf, elttype, end); + } + } + else + { + if (!is_gimple_mem_ref_addr (buf.base)) + { + buf.base = make_ssa_name (TREE_TYPE (ptr)); + gimple *g = gimple_build_assign (buf.base, ptr); + gimple_set_location (g, loc); + gsi_insert_before (gsi, g, GSI_SAME_STMT); + } + buf.alias_type = build_pointer_type (type); + clear_padding_type (&buf, type, buf.sz); + clear_padding_flush (&buf, true); + } + + gimple_stmt_iterator gsiprev2 = *gsi; + gsi_prev (&gsiprev2); + if (gsi_stmt (gsiprev) == gsi_stmt (gsiprev2)) + gsi_replace (gsi, gimple_build_nop (), true); + else + { + gsi_remove (gsi, true); + *gsi = gsiprev2; + } + return true; +} + /* Fold the non-target builtin at *GSI and return whether any simplification was made. */ @@ -4105,6 +4797,9 @@ gimple_fold_builtin (gimple_stmt_iterator *gsi) case BUILT_IN_REALLOC: return gimple_fold_builtin_realloc (gsi); + case BUILT_IN_CLEAR_PADDING: + return gimple_fold_builtin_clear_padding (gsi); + default:; } -- cgit v1.1 From 4adfcea0a1b0c6dcaefddca3d5f45dd5403b1a80 Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Tue, 24 Nov 2020 10:44:32 +0100 Subject: middle-end: Prefer no RMW in __builtin_clear_padding implementation where possible Currently the __builtin_clear_padding expansion code emits no code for full words that don't have any padding bits, and most of the time if the only padding bytes are from the start of the word it attempts to merge them with previous padding store (via {}) or if the only padding bytes are from the end of the word, it attempts to merge it with following padding bytes. For everything else it was using a RMW, except when it found an aligned char/short/int covering all the padding bytes and all those padding bytes were all ones in that store. The following patch changes it, such that we only use RMW if the padding has any bytes which have some padding and some non-padding bits (i.e. bitfields are involved), often it is the same amount of instructions in the end and avoids being thread-unsafe unless necessary (and avoids having to wait for the reads to make it into the CPU). So, if there are no bitfields, the function will just store some zero bytes, shorts, ints, long longs etc. where needed. 2020-11-24 Jakub Jelinek * gimple-fold.c (clear_padding_flush): If a word contains only 0 or 0xff bytes of padding other than all set, all clear, all set followed by all clear or all clear followed by all set, don't emit a RMW operation on the whole word or parts of it, but instead clear the individual bytes of padding. For paddings of one byte size, don't use char[1] and {}, but instead just char and 0. --- gcc/gimple-fold.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 7 deletions(-) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index 5d60bef..905c0a0 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -4033,7 +4033,9 @@ clear_padding_flush (clear_padding_struct *buf, bool full) { size_t nonzero_first = wordsize; size_t nonzero_last = 0; - bool all_ones = true; + size_t zero_first = wordsize; + size_t zero_last = 0; + bool all_ones = true, bytes_only = true; if ((unsigned HOST_WIDE_INT) (buf->off + i + wordsize) > (unsigned HOST_WIDE_INT) buf->sz) { @@ -4055,9 +4057,19 @@ clear_padding_flush (clear_padding_struct *buf, bool full) all_ones = false; nonzero_last = j + 1 - i; } + else + { + if (zero_first == wordsize) + zero_first = j - i; + zero_last = j + 1 - i; + } if (buf->buf[j] != 0 && buf->buf[j] != (unsigned char) ~0) - all_ones = false; + { + all_ones = false; + bytes_only = false; + } } + size_t padding_end = i; if (padding_bytes) { if (nonzero_first == 0 @@ -4069,7 +4081,6 @@ clear_padding_flush (clear_padding_struct *buf, bool full) padding_bytes += wordsize; continue; } - size_t padding_end = i; if (all_ones && nonzero_first == 0) { padding_bytes += nonzero_last; @@ -4077,12 +4088,27 @@ clear_padding_flush (clear_padding_struct *buf, bool full) nonzero_first = wordsize; nonzero_last = 0; } - tree atype = build_array_type_nelts (char_type_node, padding_bytes); + else if (bytes_only && nonzero_first == 0) + { + gcc_assert (zero_first && zero_first != wordsize); + padding_bytes += zero_first; + padding_end += zero_first; + } + tree atype, src; + if (padding_bytes == 1) + { + atype = char_type_node; + src = build_zero_cst (char_type_node); + } + else + { + atype = build_array_type_nelts (char_type_node, padding_bytes); + src = build_constructor (atype, NULL); + } tree dst = build2_loc (buf->loc, MEM_REF, atype, buf->base, build_int_cst (buf->alias_type, buf->off + padding_end - padding_bytes)); - tree src = build_constructor (atype, NULL); gimple *g = gimple_build_assign (dst, src); gimple_set_location (g, buf->loc); gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); @@ -4099,6 +4125,45 @@ clear_padding_flush (clear_padding_struct *buf, bool full) padding_bytes = nonzero_last - nonzero_first; continue; } + if (bytes_only) + { + /* If bitfields aren't involved in this word, prefer storing + individual bytes or groups of them over performing a RMW + operation on the whole word. */ + gcc_assert (i + zero_last <= end); + for (size_t j = padding_end; j < i + zero_last; j++) + { + if (buf->buf[j]) + { + size_t k; + for (k = j; k < i + zero_last; k++) + if (buf->buf[k] == 0) + break; + HOST_WIDE_INT off = buf->off + j; + tree atype, src; + if (k - j == 1) + { + atype = char_type_node; + src = build_zero_cst (char_type_node); + } + else + { + atype = build_array_type_nelts (char_type_node, k - j); + src = build_constructor (atype, NULL); + } + tree dst = build2_loc (buf->loc, MEM_REF, atype, + buf->base, + build_int_cst (buf->alias_type, off)); + gimple *g = gimple_build_assign (dst, src); + gimple_set_location (g, buf->loc); + gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); + j = k; + } + } + if (nonzero_last == wordsize) + padding_bytes = nonzero_last - zero_last; + continue; + } for (size_t eltsz = 1; eltsz <= wordsize; eltsz <<= 1) { if (nonzero_last - nonzero_first <= eltsz @@ -4153,12 +4218,21 @@ clear_padding_flush (clear_padding_struct *buf, bool full) { if (padding_bytes) { - tree atype = build_array_type_nelts (char_type_node, padding_bytes); + tree atype, src; + if (padding_bytes == 1) + { + atype = char_type_node; + src = build_zero_cst (char_type_node); + } + else + { + atype = build_array_type_nelts (char_type_node, padding_bytes); + src = build_constructor (atype, NULL); + } tree dst = build2_loc (buf->loc, MEM_REF, atype, buf->base, build_int_cst (buf->alias_type, buf->off + end - padding_bytes)); - tree src = build_constructor (atype, NULL); gimple *g = gimple_build_assign (dst, src); gimple_set_location (g, buf->loc); gsi_insert_before (buf->gsi, g, GSI_SAME_STMT); -- cgit v1.1 From a7285c8659689e9ade53fcb996b26d874455a4f3 Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Wed, 25 Nov 2020 10:37:58 +0100 Subject: middle-end: Reject flexible array members in __builtin_clear_padding [PR97943] As mentioned in the PR, we currently ICE on flexible array members in structs and unions during __builtin_clear_padding processing. Jason said in the PR he'd prefer an error in these cases over forcefully handling it as [0] arrays (everything is padding then) or consider the arrays to have as many whole elements as would fit into the tail padding. So, this patch implements that. 2020-11-25 Jakub Jelinek PR middle-end/97943 * gimple-fold.c (clear_padding_union, clear_padding_type): Error on and ignore flexible array member fields. Ignore fields with error_mark_node type. * c-c++-common/builtin-clear-padding-2.c: New test. * c-c++-common/builtin-clear-padding-3.c: New test. * g++.dg/ext/builtin-clear-padding-1.C: New test. * gcc.dg/builtin-clear-padding-2.c: New test. --- gcc/gimple-fold.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index 905c0a0..1f0a6097 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -4329,6 +4329,17 @@ clear_padding_union (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) if (TREE_CODE (field) == FIELD_DECL) { + if (DECL_SIZE_UNIT (field) == NULL_TREE) + { + if (TREE_TYPE (field) == error_mark_node) + continue; + gcc_assert (TREE_CODE (TREE_TYPE (field)) == ARRAY_TYPE + && !COMPLETE_TYPE_P (TREE_TYPE (field))); + error_at (buf->loc, "flexible array member %qD does not have " + "well defined padding bits for %qs", + field, "__builtin_clear_padding"); + continue; + } HOST_WIDE_INT fldsz = tree_to_shwi (DECL_SIZE_UNIT (field)); gcc_assert (union_buf->size == 0); union_buf->off = start_off; @@ -4446,11 +4457,12 @@ clear_padding_type (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) if (TREE_CODE (field) == FIELD_DECL) { + tree ftype = TREE_TYPE (field); if (DECL_BIT_FIELD (field)) { if (DECL_NAME (field) == NULL_TREE) continue; - HOST_WIDE_INT fldsz = TYPE_PRECISION (TREE_TYPE (field)); + HOST_WIDE_INT fldsz = TYPE_PRECISION (ftype); if (fldsz == 0) continue; HOST_WIDE_INT pos = int_byte_position (field); @@ -4513,6 +4525,16 @@ clear_padding_type (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) } } } + else if (DECL_SIZE_UNIT (field) == NULL_TREE) + { + if (ftype == error_mark_node) + continue; + gcc_assert (TREE_CODE (ftype) == ARRAY_TYPE + && !COMPLETE_TYPE_P (ftype)); + error_at (buf->loc, "flexible array member %qD does not have " + "well defined padding bits for %qs", + field, "__builtin_clear_padding"); + } else { HOST_WIDE_INT pos = int_byte_position (field); -- cgit v1.1 From 93a732514865f3607cc01f5c5b078f63580ef4b1 Mon Sep 17 00:00:00 2001 From: Matthew Malcomson Date: Wed, 25 Nov 2020 16:31:47 +0000 Subject: libsanitizer: Add hwasan pass and associated gimple changes There are four main features to this change: 1) Check pointer tags match address tags. When sanitizing for hwasan we now put HWASAN_CHECK internal functions before memory accesses in the `asan` pass. This checks that a tag in the pointer being used match the tag stored in shadow memory for the memory region being used. These internal functions are expanded into actual checks in the sanopt pass that happens just before expansion into RTL. We use the same mechanism that currently inserts ASAN_CHECK internal functions to insert the new HWASAN_CHECK functions. 2) Instrument known builtin function calls. Handle all builtin functions that we know use memory accesses. This commit uses the machinery added for ASAN to identify builtin functions that access memory. The main differences between the approaches for HWASAN and ASAN are: - libhwasan intercepts much less builtin functions. - Alloca needs to be transformed differently (instead of adding redzones it needs to tag shadow memory and return a tagged pointer). - stack_restore needs to untag the shadow stack between the current position and where it's going. - `noreturn` functions can not be handled by simply unpoisoning the entire shadow stack -- there is no "always valid" tag. (exceptions and things such as longjmp need to be handled in a different way, usually in the runtime). For hardware implemented checking (such as AArch64's memory tagging extension) alloca and stack_restore will need to be handled by hooks in the backend rather than transformation at the gimple level. This will allow architecture specific handling of such stack modifications. 3) Introduce HWASAN block-scope poisoning Here we use exactly the same mechanism as ASAN_MARK to poison/unpoison variables on entry/exit of a block. In order to simply use the exact same machinery we're using the same internal functions until the SANOPT pass. This means that all handling of ASAN_MARK is the same. This has the negative that the naming may be a little confusing, but a positive that handling of the internal function doesn't have to be duplicated for a function that behaves exactly the same but has a different name. gcc/ChangeLog: * asan.c (asan_instrument_reads): New. (asan_instrument_writes): New. (asan_memintrin): New. (handle_builtin_stack_restore): Account for HWASAN. (handle_builtin_alloca): Account for HWASAN. (get_mem_refs_of_builtin_call): Special case strlen for HWASAN. (hwasan_instrument_reads): New. (hwasan_instrument_writes): New. (hwasan_memintrin): New. (report_error_func): Assert not HWASAN. (build_check_stmt): Make HWASAN_CHECK instead of ASAN_CHECK. (instrument_derefs): HWASAN does not tag globals. (instrument_builtin_call): Use new helper functions. (maybe_instrument_call): Don't instrument `noreturn` functions. (initialize_sanitizer_builtins): Add new type. (asan_expand_mark_ifn): Account for HWASAN. (asan_expand_check_ifn): Assert never called by HWASAN. (asan_expand_poison_ifn): Account for HWASAN. (asan_instrument): Branch based on whether using HWASAN or ASAN. (pass_asan::gate): Return true if sanitizing HWASAN. (pass_asan_O0::gate): Return true if sanitizing HWASAN. (hwasan_check_func): New. (hwasan_expand_check_ifn): New. (hwasan_expand_mark_ifn): New. (gate_hwasan): New. * asan.h (hwasan_expand_check_ifn): New decl. (hwasan_expand_mark_ifn): New decl. (gate_hwasan): New decl. (asan_intercepted_p): Always false for hwasan. (asan_sanitize_use_after_scope): Account for HWASAN. * builtin-types.def (BT_FN_PTR_CONST_PTR_UINT8): New. * gimple-fold.c (gimple_build): New overload for building function calls without arguments. (gimple_build_round_up): New. * gimple-fold.h (gimple_build): New decl. (gimple_build): New inline function. (gimple_build_round_up): New decl. (gimple_build_round_up): New inline function. * gimple-pretty-print.c (dump_gimple_call_args): Account for HWASAN. * gimplify.c (asan_poison_variable): Account for HWASAN. (gimplify_function_tree): Remove requirement of SANITIZE_ADDRESS, requiring asan or hwasan is accounted for in `asan_sanitize_use_after_scope`. * internal-fn.c (expand_HWASAN_CHECK): New. (expand_HWASAN_ALLOCA_UNPOISON): New. (expand_HWASAN_CHOOSE_TAG): New. (expand_HWASAN_MARK): New. (expand_HWASAN_SET_TAG): New. * internal-fn.def (HWASAN_ALLOCA_UNPOISON): New. (HWASAN_CHOOSE_TAG): New. (HWASAN_CHECK): New. (HWASAN_MARK): New. (HWASAN_SET_TAG): New. * sanitizer.def (BUILT_IN_HWASAN_LOAD1): New. (BUILT_IN_HWASAN_LOAD2): New. (BUILT_IN_HWASAN_LOAD4): New. (BUILT_IN_HWASAN_LOAD8): New. (BUILT_IN_HWASAN_LOAD16): New. (BUILT_IN_HWASAN_LOADN): New. (BUILT_IN_HWASAN_STORE1): New. (BUILT_IN_HWASAN_STORE2): New. (BUILT_IN_HWASAN_STORE4): New. (BUILT_IN_HWASAN_STORE8): New. (BUILT_IN_HWASAN_STORE16): New. (BUILT_IN_HWASAN_STOREN): New. (BUILT_IN_HWASAN_LOAD1_NOABORT): New. (BUILT_IN_HWASAN_LOAD2_NOABORT): New. (BUILT_IN_HWASAN_LOAD4_NOABORT): New. (BUILT_IN_HWASAN_LOAD8_NOABORT): New. (BUILT_IN_HWASAN_LOAD16_NOABORT): New. (BUILT_IN_HWASAN_LOADN_NOABORT): New. (BUILT_IN_HWASAN_STORE1_NOABORT): New. (BUILT_IN_HWASAN_STORE2_NOABORT): New. (BUILT_IN_HWASAN_STORE4_NOABORT): New. (BUILT_IN_HWASAN_STORE8_NOABORT): New. (BUILT_IN_HWASAN_STORE16_NOABORT): New. (BUILT_IN_HWASAN_STOREN_NOABORT): New. (BUILT_IN_HWASAN_TAG_MISMATCH4): New. (BUILT_IN_HWASAN_HANDLE_LONGJMP): New. (BUILT_IN_HWASAN_TAG_PTR): New. * sanopt.c (sanopt_optimize_walker): Act for hwasan. (pass_sanopt::execute): Act for hwasan. * toplev.c (compile_file): Use `gate_hwasan` function. --- gcc/gimple-fold.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index 1f0a6097..40fc7e4 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -8479,6 +8479,32 @@ gimple_build (gimple_seq *seq, location_t loc, return res; } +/* Build the call FN () with a result of type TYPE (or no result if TYPE is + void) with a location LOC. Returns the built expression value (or NULL_TREE + if TYPE is void) and appends statements possibly defining it to SEQ. */ + +tree +gimple_build (gimple_seq *seq, location_t loc, combined_fn fn, tree type) +{ + tree res = NULL_TREE; + gcall *stmt; + if (internal_fn_p (fn)) + stmt = gimple_build_call_internal (as_internal_fn (fn), 0); + else + { + tree decl = builtin_decl_implicit (as_builtin_fn (fn)); + stmt = gimple_build_call (decl, 0); + } + if (!VOID_TYPE_P (type)) + { + res = create_tmp_reg_or_ssa_name (type); + gimple_call_set_lhs (stmt, res); + } + gimple_set_location (stmt, loc); + gimple_seq_add_stmt_without_update (seq, stmt); + return res; +} + /* Build the call FN (ARG0) with a result of type TYPE (or no result if TYPE is void) with location LOC, simplifying it first if possible. Returns the built @@ -8668,6 +8694,26 @@ gimple_build_vector (gimple_seq *seq, location_t loc, return builder->build (); } +/* Emit gimple statements into &stmts that take a value given in OLD_SIZE + and generate a value guaranteed to be rounded upwards to ALIGN. + + Return the tree node representing this size, it is of TREE_TYPE TYPE. */ + +tree +gimple_build_round_up (gimple_seq *seq, location_t loc, tree type, + tree old_size, unsigned HOST_WIDE_INT align) +{ + unsigned HOST_WIDE_INT tg_mask = align - 1; + /* tree new_size = (old_size + tg_mask) & ~tg_mask; */ + gcc_assert (INTEGRAL_TYPE_P (type)); + tree tree_mask = build_int_cst (type, tg_mask); + tree oversize = gimple_build (seq, loc, PLUS_EXPR, type, old_size, + tree_mask); + + tree mask = build_int_cst (type, -align); + return gimple_build (seq, loc, BIT_AND_EXPR, type, oversize, mask); +} + /* Return true if the result of assignment STMT is known to be non-negative. If the return value is based on the assumption that signed overflow is undefined, set *STRICT_OVERFLOW_P to true; otherwise, don't change -- cgit v1.1 From aec2d6849160f92cd45f97d6c3bdd8808ab01fa6 Mon Sep 17 00:00:00 2001 From: Martin Sebor Date: Wed, 25 Nov 2020 11:00:10 -0700 Subject: PR middle-end/97956 - ICE due to type mismatch in pointer_plus_expr during memchr folding gcc/ChangeLog: PR middle-end/97956 * gimple-fold.c (gimple_fold_builtin_memchr): Use sizetype for pointer offsets. gcc/testsuite/ChangeLog: PR middle-end/97956 * gcc.dg/memchr-3.c: New test. --- gcc/gimple-fold.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index 40fc7e4..58b6ea4 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -2689,7 +2689,7 @@ gimple_fold_builtin_memchr (gimple_stmt_iterator *gsi) gimple_seq stmts = NULL; if (lhs != NULL_TREE) { - tree offset_cst = build_int_cst (TREE_TYPE (len), offset); + tree offset_cst = build_int_cst (sizetype, offset); gassign *stmt = gimple_build_assign (lhs, POINTER_PLUS_EXPR, arg1, offset_cst); gimple_seq_add_stmt_without_update (&stmts, stmt); -- cgit v1.1 From a386566118054e08bb733f1248649fb6847c407e Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Thu, 26 Nov 2020 10:51:51 +0100 Subject: gimple-fold: Use DECL_PADDING_P in __builtin_clear_padding On Wed, Nov 25, 2020 at 12:26:17PM -0500, Jason Merrill wrote: > I think you want to check DECL_PADDING_P here; the C and C++ front ends set > it on unnamed bit-fields, and that's what is_empty_type looks at. While the above has been written in the context of __builtin_bit_cast patch, I think it applies to __builtin_clear_padding too. So this patch implements that. The C FE sets DECL_PADDING_P solely on the DECL_BIT_FIELD !DECL_NAME FIELD_DECLs, the C++ FE sets it on those and in another spot I haven't really figured out what it is about. 2020-11-26 Jakub Jelinek * gimple-fold.c (clear_padding_union): Ignore DECL_PADDING_P fields. (clear_padding_type): Ignore DECL_PADDING_P fields, rather than DECL_BIT_FIELD with NULL DECL_NAME. --- gcc/gimple-fold.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index 58b6ea4..a821b64 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -4327,7 +4327,7 @@ clear_padding_union (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) } for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) - if (TREE_CODE (field) == FIELD_DECL) + if (TREE_CODE (field) == FIELD_DECL && !DECL_PADDING_P (field)) { if (DECL_SIZE_UNIT (field) == NULL_TREE) { @@ -4455,13 +4455,11 @@ clear_padding_type (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) HOST_WIDE_INT cur_pos; cur_pos = 0; for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) - if (TREE_CODE (field) == FIELD_DECL) + if (TREE_CODE (field) == FIELD_DECL && !DECL_PADDING_P (field)) { tree ftype = TREE_TYPE (field); if (DECL_BIT_FIELD (field)) { - if (DECL_NAME (field) == NULL_TREE) - continue; HOST_WIDE_INT fldsz = TYPE_PRECISION (ftype); if (fldsz == 0) continue; -- cgit v1.1 From bf0a63a1f47525d1c466dbb84616dcb72010affa Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Fri, 27 Nov 2020 11:23:45 +0100 Subject: gimple-fold: Fix another __builtin_clear_padding ICE When playing with __builtin_bit_cast, I have noticed __builtin_clear_padding ICE on the G class below. The artificial field with D type has offset 0 and size 8 bytes, but the following artificial field with E type has offset 0 and size 0, so it triggers the asserts that we don't move current position backwards. Fixed by ignoring is_empty_type (TREE_TYPE (field)) fields, all of their bits are padding which is what is added when skipping over to next field anyway. 2020-11-27 Jakub Jelinek PR libstdc++/88101 * gimple-fold.c (clear_padding_type): Ignore fields with is_empty_type types. * g++.dg/torture/builtin-clear-padding-3.C: New test. --- gcc/gimple-fold.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index a821b64..1f3d80e 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -4533,6 +4533,8 @@ clear_padding_type (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) "well defined padding bits for %qs", field, "__builtin_clear_padding"); } + else if (is_empty_type (TREE_TYPE (field))) + continue; else { HOST_WIDE_INT pos = int_byte_position (field); -- cgit v1.1 From 337d6362458ab033d3bfe287dda37f9da5577406 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Wed, 2 Dec 2020 09:44:40 +0100 Subject: Fix __builtin_clear_padding for empty struct. gcc/ChangeLog: PR c/98087 * gimple-fold.c (clear_padding_type): Do not divide by zero. gcc/testsuite/ChangeLog: PR c/98087 * gcc.c-torture/compile/pr98087.c: New test. --- gcc/gimple-fold.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index 1f3d80e..ab74494 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -4552,6 +4552,8 @@ clear_padding_type (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) case ARRAY_TYPE: HOST_WIDE_INT nelts, fldsz; fldsz = int_size_in_bytes (TREE_TYPE (type)); + if (fldsz == 0) + break; nelts = sz / fldsz; if (nelts > 1 && sz > 8 * UNITS_PER_WORD -- cgit v1.1 From 896048cf43d5eb21ab7c16553bb9d13b0f890b81 Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Thu, 3 Dec 2020 15:46:54 +0100 Subject: c++: Add __builtin_bit_cast to implement std::bit_cast [PR93121] The following patch adds __builtin_bit_cast builtin, similarly to clang or MSVC which implement std::bit_cast using such an builtin too. It checks the various std::bit_cast requirements, when not constexpr evaluated acts pretty much like VIEW_CONVERT_EXPR of the source argument to the destination type and the hardest part is obviously the constexpr evaluation. I've left out PDP11 handling of those, couldn't figure out how exactly are bitfields laid out there 2020-12-03 Jakub Jelinek PR libstdc++/93121 * fold-const.h (native_encode_initializer): Add mask argument defaulted to nullptr. (find_bitfield_repr_type): Declare. (native_interpret_aggregate): Declare. * fold-const.c (find_bitfield_repr_type): New function. (native_encode_initializer): Add mask argument and support for filling it. Handle also some bitfields without integral DECL_BIT_FIELD_REPRESENTATIVE. (native_interpret_aggregate): New function. * gimple-fold.h (clear_type_padding_in_mask): Declare. * gimple-fold.c (struct clear_padding_struct): Add clear_in_mask member. (clear_padding_flush): Handle buf->clear_in_mask. (clear_padding_union): Copy clear_in_mask. Don't error if buf->clear_in_mask is set. (clear_padding_type): Don't error if buf->clear_in_mask is set. (clear_type_padding_in_mask): New function. (gimple_fold_builtin_clear_padding): Set buf.clear_in_mask to false. * doc/extend.texi (__builtin_bit_cast): Document. * c-common.h (enum rid): Add RID_BUILTIN_BIT_CAST. * c-common.c (c_common_reswords): Add __builtin_bit_cast. * cp-tree.h (cp_build_bit_cast): Declare. * cp-tree.def (BIT_CAST_EXPR): New tree code. * cp-objcp-common.c (names_builtin_p): Handle RID_BUILTIN_BIT_CAST. (cp_common_init_ts): Handle BIT_CAST_EXPR. * cxx-pretty-print.c (cxx_pretty_printer::postfix_expression): Likewise. * parser.c (cp_parser_postfix_expression): Handle RID_BUILTIN_BIT_CAST. * semantics.c (cp_build_bit_cast): New function. * tree.c (cp_tree_equal): Handle BIT_CAST_EXPR. (cp_walk_subtrees): Likewise. * pt.c (tsubst_copy): Likewise. * constexpr.c (check_bit_cast_type, cxx_eval_bit_cast): New functions. (cxx_eval_constant_expression): Handle BIT_CAST_EXPR. (potential_constant_expression_1): Likewise. * cp-gimplify.c (cp_genericize_r): Likewise. * g++.dg/cpp2a/bit-cast1.C: New test. * g++.dg/cpp2a/bit-cast2.C: New test. * g++.dg/cpp2a/bit-cast3.C: New test. * g++.dg/cpp2a/bit-cast4.C: New test. * g++.dg/cpp2a/bit-cast5.C: New test. --- gcc/gimple-fold.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 6 deletions(-) (limited to 'gcc/gimple-fold.c') diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c index ab74494..3148c6b 100644 --- a/gcc/gimple-fold.c +++ b/gcc/gimple-fold.c @@ -3958,6 +3958,10 @@ static const size_t clear_padding_buf_size = 32 * clear_padding_unit; /* Data passed through __builtin_clear_padding folding. */ struct clear_padding_struct { location_t loc; + /* 0 during __builtin_clear_padding folding, nonzero during + clear_type_padding_in_mask. In that case, instead of clearing the + non-padding bits in union_ptr array clear the padding bits in there. */ + bool clear_in_mask; tree base; tree alias_type; gimple_stmt_iterator *gsi; @@ -4000,6 +4004,39 @@ clear_padding_flush (clear_padding_struct *buf, bool full) size_t padding_bytes = buf->padding_bytes; if (buf->union_ptr) { + if (buf->clear_in_mask) + { + /* During clear_type_padding_in_mask, clear the padding + bits set in buf->buf in the buf->union_ptr mask. */ + for (size_t i = 0; i < end; i++) + { + if (buf->buf[i] == (unsigned char) ~0) + padding_bytes++; + else + { + memset (&buf->union_ptr[buf->off + i - padding_bytes], + 0, padding_bytes); + padding_bytes = 0; + buf->union_ptr[buf->off + i] &= ~buf->buf[i]; + } + } + if (full) + { + memset (&buf->union_ptr[buf->off + end - padding_bytes], + 0, padding_bytes); + buf->off = 0; + buf->size = 0; + buf->padding_bytes = 0; + } + else + { + memmove (buf->buf, buf->buf + end, buf->size - end); + buf->off += end; + buf->size -= end; + buf->padding_bytes = padding_bytes; + } + return; + } /* Inside of a union, instead of emitting any code, instead clear all bits in the union_ptr buffer that are clear in buf. Whole padding bytes don't clear anything. */ @@ -4311,6 +4348,7 @@ clear_padding_union (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) clear_padding_flush (buf, false); union_buf = XALLOCA (clear_padding_struct); union_buf->loc = buf->loc; + union_buf->clear_in_mask = buf->clear_in_mask; union_buf->base = NULL_TREE; union_buf->alias_type = NULL_TREE; union_buf->gsi = NULL; @@ -4335,9 +4373,10 @@ clear_padding_union (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) continue; gcc_assert (TREE_CODE (TREE_TYPE (field)) == ARRAY_TYPE && !COMPLETE_TYPE_P (TREE_TYPE (field))); - error_at (buf->loc, "flexible array member %qD does not have " - "well defined padding bits for %qs", - field, "__builtin_clear_padding"); + if (!buf->clear_in_mask) + error_at (buf->loc, "flexible array member %qD does not have " + "well defined padding bits for %qs", + field, "__builtin_clear_padding"); continue; } HOST_WIDE_INT fldsz = tree_to_shwi (DECL_SIZE_UNIT (field)); @@ -4529,9 +4568,10 @@ clear_padding_type (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) continue; gcc_assert (TREE_CODE (ftype) == ARRAY_TYPE && !COMPLETE_TYPE_P (ftype)); - error_at (buf->loc, "flexible array member %qD does not have " - "well defined padding bits for %qs", - field, "__builtin_clear_padding"); + if (!buf->clear_in_mask) + error_at (buf->loc, "flexible array member %qD does not " + "have well defined padding bits for %qs", + field, "__builtin_clear_padding"); } else if (is_empty_type (TREE_TYPE (field))) continue; @@ -4645,6 +4685,27 @@ clear_padding_type (clear_padding_struct *buf, tree type, HOST_WIDE_INT sz) } } +/* Clear padding bits of TYPE in MASK. */ + +void +clear_type_padding_in_mask (tree type, unsigned char *mask) +{ + clear_padding_struct buf; + buf.loc = UNKNOWN_LOCATION; + buf.clear_in_mask = true; + buf.base = NULL_TREE; + buf.alias_type = NULL_TREE; + buf.gsi = NULL; + buf.align = 0; + buf.off = 0; + buf.padding_bytes = 0; + buf.sz = int_size_in_bytes (type); + buf.size = 0; + buf.union_ptr = mask; + clear_padding_type (&buf, type, buf.sz); + clear_padding_flush (&buf, true); +} + /* Fold __builtin_clear_padding builtin. */ static bool @@ -4664,6 +4725,7 @@ gimple_fold_builtin_clear_padding (gimple_stmt_iterator *gsi) gsi_prev (&gsiprev); buf.loc = loc; + buf.clear_in_mask = false; buf.base = ptr; buf.alias_type = NULL_TREE; buf.gsi = gsi; -- cgit v1.1